version 2 saverds

npct · Jun 27, 2019 · 96edd13 · 96edd13
1 parent 57a952a
commit 96edd13
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 66 deletions.
diff --git a/04a_cyclestreets_master.R b/04a_cyclestreets_master.R
@@ -100,7 +100,7 @@ if (purpose=="commute")  {
 }
 lines_cs <- SpatialLinesDataFrame(sl = lines_cs_lines, data = lines_cs_data)
 proj4string(lines_cs) <- proj_4326
-saveRDS(lines_cs, (file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "lines_cs.Rds")))
+saveRDS(lines_cs, (file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "lines_cs.Rds")), version = 2)
 
 #########################
 ### PARTS 2 AND 3: ROUTING 
@@ -134,5 +134,5 @@ rfrq_all_data$dist_rf_e  <- rfrq_all_data$rf_dist_km / rfrq_all_data$e_dist_km
 rfrq_all_data$dist_rq_rf <- rfrq_all_data$rq_dist_km / rfrq_all_data$rf_dist_km
 rfrq_all_data <- rfrq_all_data[, c(1:4, 5, 8, 11, 12, 6, 9, 7, 10)]
 write_csv(rfrq_all_data, file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rfrq_all_data.csv"))
-# saveRDS(rfrq_all_data,file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rfrq_all_data.Rds"))
+# saveRDS(rfrq_all_data,file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rfrq_all_data.Rds"), version = 2)
 
diff --git a/04b_cyclestreets_batch_routing.R b/04b_cyclestreets_batch_routing.R
@@ -26,7 +26,7 @@ for(i in 1:nbatch){
   routes <- line2route(lines_cs_sub, route_fun = route_cyclestreet, plan = route_type, n_processes = 10, base_url = "http://pct.cyclestreets.net/api/")
   routes@data <- routes@data[,!names(routes@data) %in% c("plan","start","finish")] # drop fields not wanted
   routes@data <- left_join(routes@data, lines_cs_sub@data, by = "id")  # merge in data in lines file
-  saveRDS(routes,file = file.path(path_temp_cs, purpose, geography, paste0("r",substr(route_type, 1, 1),"_",file_name,"_",i,"of",nbatch,".Rds")))
+  saveRDS(routes,file = file.path(path_temp_cs, purpose, geography, paste0("r",substr(route_type, 1, 1),"_",file_name,"_",i,"of",nbatch,".Rds")), version = 2)
   print(paste0("Batch ",i," of ",nbatch," finished at ",Sys.time()))
 }
 
@@ -73,7 +73,7 @@ if(any(is.na(stack_data$length))) {
   routes_redo@data <- left_join(routes_redo@data, stack_redo@data, by = "id")
   routes_redo <- routes_redo[routes_redo@data$id!=id1,] # REMOVE THE FIRST LINE, THAT WAS REPEATED JUST TO MAKE SURE LINE2ROUTE RAN OK
   row.names(routes_redo@data) <- sapply(1:length(routes_redo), function(j) routes_redo@lines[[j]]@ID)
-  saveRDS(routes_redo,file = file.path(path_temp_cs, purpose, geography, paste0("r",substr(route_type, 1, 1),"_",file_name,"_redo_of",nbatch,".Rds")))
+  saveRDS(routes_redo,file = file.path(path_temp_cs, purpose, geography, paste0("r",substr(route_type, 1, 1),"_",file_name,"_redo_of",nbatch,".Rds")), version = 2)
   routes_redo_data <- routes_redo@data
   stack_data <- rbind(stack_keep, routes_redo_data)
   stack_redo2 <- stack_data[is.na(stack_data$length) | is.na(stack_data$av_incline) | is.na(stack_data$time) | !is.na(stack_data$error), ]
@@ -84,7 +84,7 @@ if(any(is.na(stack_data$length))) {
 if (route_type == "fastest") {
   stack_data <- stack_data[(stack_data$length < (maxdist_scenario * 1000)),]  # NB length in metres, maxdist_scenario in km
 }
-saveRDS(stack_data, file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("archive/r",substr(route_type, 1, 1),"_",file_name,"_data.Rds")))
+saveRDS(stack_data, file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("archive/r",substr(route_type, 1, 1),"_",file_name,"_data.Rds")), version = 2)
 
 # REJOIN THE FILES FOR **ROUTES** (FASTEST ROUTE LENGTH < VISUALISE DISTANCE), MERGE IN ROUTES THAT INITIALLY FAILED, & SAVE SHAPE
 rf_data_visualise <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("archive/rf_",file_name,"_data.Rds")))
@@ -138,5 +138,5 @@ if(file.exists(file.path(path_temp_cs, purpose, geography, paste0("r", substr(ro
 }
 stack@data <- stack@data["id"]
 
-saveRDS(stack,file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("archive/r",substr(route_type, 1, 1),"_",file_name,"_shape.Rds")))
-saveRDS(stack,file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("r",substr(route_type, 1, 1),"_shape.Rds")))
+saveRDS(stack,file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("archive/r",substr(route_type, 1, 1),"_",file_name,"_shape.Rds")), version = 2)
+saveRDS(stack,file = file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, paste0("r",substr(route_type, 1, 1),"_shape.Rds")), version = 2)
diff --git a/05.3_alltrips_scenario_building_bodge.R b/05.3_alltrips_scenario_building_bodge.R
@@ -18,12 +18,12 @@ rnet_codebook <- read_csv(file.path(path_outputs_regional_R, purpose, geography,
 z <- readRDS(file.path(path_outputs_regional_R, purpose, geography, region,  "github_originals/z.Rds"))
 z@data <- dplyr::rename(z@data, geo_name = `geo_label`)
 z@data <- z@data[z_codebook$`Variable name`]
-saveRDS(z, file.path(path_outputs_regional_R, purpose, geography, region,  "z.Rds"))
+saveRDS(z, file.path(path_outputs_regional_R, purpose, geography, region,  "z.Rds"), version = 2)
 
 c <- readRDS(file.path(path_outputs_regional_R, purpose, geography, region,  "github_originals/c.Rds"))
 c@data <- dplyr::rename(c@data, geo_name = `geo_label`)
 c@data <- c@data[c_codebook$`Variable name`]
-saveRDS(c, file.path(path_outputs_regional_R, purpose, geography, region,  "c.Rds"))
+saveRDS(c, file.path(path_outputs_regional_R, purpose, geography, region,  "c.Rds"), version = 2)
 
 l <- readRDS(file.path(path_outputs_regional_R, purpose, geography, region,  "github_originals/l.Rds"))
 l@data <- dplyr::rename(l@data, geo_code1 = `msoa1`, geo_code2 = `msoa2`, geo_name1 = `geo_label1`, 
@@ -32,41 +32,21 @@ l@data <- dplyr::rename(l@data, geo_code1 = `msoa1`, geo_code2 = `msoa2`, geo_na
                         dist_rq_rf = `distq_f`, rf_avslope_perc = `avslope`,
                         rq_avslope_perc = `avslope_q`,  rf_time_min = `time_fast`, rq_time_min = `time_quiet`)
 l@data <- l@data[od_l_rf_codebook$`Variable name`]
-saveRDS(l, file.path(path_outputs_regional_R, purpose, geography, region,  "l.Rds"))
+saveRDS(l, file.path(path_outputs_regional_R, purpose, geography, region,  "l.Rds"), version = 2)
 
 rf <- readRDS(file.path(path_outputs_regional_R, purpose, geography, region,  "github_originals/rf.Rds"))
 rf@data <- left_join(rf@data, l@data, by="id")
 rf@data <- rf@data[od_l_rf_codebook$`Variable name`]
-saveRDS(rf, file.path(path_outputs_regional_R, purpose, geography, region,  "rf.Rds"))
+saveRDS(rf, file.path(path_outputs_regional_R, purpose, geography, region,  "rf.Rds"), version = 2)
 
 rq <- readRDS(file.path(path_outputs_regional_R, purpose, geography, region,  "github_originals/rq.Rds"))
 rq@data <- left_join(rq@data, l@data, by="id")
 rq@data <- rq@data[rq_codebook$`Variable name`]
-saveRDS(rq, file.path(path_outputs_regional_R, purpose, geography, region,  "rq.Rds"))
+saveRDS(rq, file.path(path_outputs_regional_R, purpose, geography, region,  "rq.Rds"), version = 2)
 
 rnet <- readRDS(file.path(path_outputs_regional_R, purpose, geography, region,  "github_originals/rnet.Rds"))
 rnet@data <- dplyr::rename(rnet@data, singlezone = `Singlezone`)
 rnet@data$local_id <- 1:nrow(rnet)
 rnet@data <- rnet@data[rnet_codebook$`Variable name`]
-saveRDS(rnet, file.path(path_outputs_regional_R, purpose, geography, region,  "rnet.Rds"))
+saveRDS(rnet, file.path(path_outputs_regional_R, purpose, geography, region,  "rnet.Rds"), version = 2)
 
-
-# ## MAKE AND ISLE OF WIGHT TEST VERSION, COPY FROM COMMUTE
-# region <- "isle-of-wight"
-# z <- readRDS(file.path(path_outputs_regional_R, "commute", geography, region,  "z.Rds"))
-# saveRDS(z, file.path(path_outputs_regional_R, purpose, geography, region,  "z.Rds"))
-# 
-# c <- readRDS(file.path(path_outputs_regional_R, "commute", geography, region,  "c.Rds"))
-# saveRDS(c, file.path(path_outputs_regional_R, purpose, geography, region,  "c.Rds"))
-# 
-# l <- readRDS(file.path(path_outputs_regional_R, "commute", geography, region,  "l.Rds"))
-# saveRDS(l, file.path(path_outputs_regional_R, purpose, geography, region,  "l.Rds"))
-# 
-# rf <- readRDS(file.path(path_outputs_regional_R, "commute", geography, region,  "rf.Rds"))
-# saveRDS(rf, file.path(path_outputs_regional_R, purpose, geography, region,  "rf.Rds"))
-# 
-# rq <- readRDS(file.path(path_outputs_regional_R, "commute", geography, region,  "rq.Rds"))
-# saveRDS(rq, file.path(path_outputs_regional_R, purpose, geography, region,  "rq.Rds"))
-# 
-# rnet <- readRDS(file.path(path_outputs_regional_R, "commute", geography, region,  "rnet.Rds"))
-# saveRDS(rnet, file.path(path_outputs_regional_R, purpose, geography, region,  "rnet.Rds"))
diff --git a/06_merge_national_datasets.R b/06_merge_national_datasets.R
@@ -32,9 +32,9 @@ z_shape_private <- z_shape
 
 if(purpose == "commute") {
   c_shape <- spTransform(c_shape, proj_4326)
-  l_shape <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "lines_cs.Rds"))
-  rf_shape <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rf_shape.Rds"))
-  rq_shape <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rq_shape.Rds"))
+  l_shape <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "lines_cs.Rds") , version = 2)
+  rf_shape <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rf_shape.Rds") , version = 2)
+  rq_shape <- readRDS(file.path(path_inputs, "02_intermediate/02_travel_data", purpose, geography, "rq_shape.Rds") , version = 2)
 }
 if(purpose == "school") {
   d_shape <- readOGR(file.path(path_inputs,"02_intermediate/01_geographies/urn_cents.geojson"))
@@ -78,7 +78,7 @@ print(summary({sel_zone <- z_shape$geo_code %in% z_all_attributes$geo_code})) #
 z_shape <- z_shape[sel_zone,]  
 z_shape@data <- data.frame(geo_code = z_shape$geo_code) 
 z_shape@data <- left_join(z_shape@data, z_all_attributes, by="geo_code")
-saveRDS(z_shape, file.path(path_outputs_national, purpose, geography, "z_all.Rds"))
+saveRDS(z_shape, file.path(path_outputs_national, purpose, geography, "z_all.Rds") , version = 2)
 geojson_write(z_shape, file = file.path(path_outputs_national, purpose, geography, "z_all.geojson"))
 
 if(purpose == "commute") {
@@ -96,7 +96,7 @@ if(purpose == "commute") {
   c_shape@data <- data.frame(geo_code = c_shape$geo_code) 
   c_shape@data <- left_join(c_shape@data, c_all_attributes, by="geo_code")
   c_shape@data <- c_shape@data[c_codebook$`Variable name`]
-  saveRDS(c_shape, file.path(path_outputs_national, purpose, geography, "c_all.Rds"))
+  saveRDS(c_shape, file.path(path_outputs_national, purpose, geography, "c_all.Rds") , version = 2)
   geojson_write(c_shape, file = (file.path(path_outputs_national, purpose, geography, "c_all.geojson")))
 
 # MERGE LINE SCENARIO DATA TO BETWEEN-ZONE LINES FILE
@@ -108,15 +108,15 @@ if(purpose == "commute") {
   l_shape <- l_shape[sel_line2,]  
   l_shape@data <- data.frame(id = l_shape$id) 
   l_shape@data <- left_join(l_shape@data, od_all_attributes, by="id")
-  saveRDS(l_shape, (file.path(path_outputs_national, purpose, geography, "l_all.Rds")))
+  saveRDS(l_shape, (file.path(path_outputs_national, purpose, geography, "l_all.Rds")) , version = 2)
 
 # MERGE LINE SCENARIO DATA TO FAST ROUTES FILE
   print("rf")
   print(summary(({sel_rf <- (rf_shape$id %in% od_all_attributes$id)}))) # Limit to those with od_attributes (minflow_visualise)
   rf_shape <- rf_shape[sel_rf,]  
   rf_shape@data <- data.frame(id = rf_shape$id) 
   rf_shape@data <- left_join(rf_shape@data, od_all_attributes, by="id")
-  saveRDS(rf_shape, (file.path(path_outputs_national, purpose, geography, "rf_all.Rds")))
+  saveRDS(rf_shape, (file.path(path_outputs_national, purpose, geography, "rf_all.Rds")) , version = 2)
 
 # MERGE LINE SCENARIO DATA TO QUIET ROUTES FILE 
   print("rq")
@@ -125,7 +125,7 @@ if(purpose == "commute") {
   rq_shape@data <- data.frame(id = rq_shape$id) 
   rq_shape@data <- left_join(rq_shape@data, od_all_attributes, by="id")
   rq_shape@data <- rq_shape@data[rq_codebook$`Variable name`]
-  saveRDS(rq_shape, (file.path(path_outputs_national, purpose, geography, "rq_all.Rds")))
+  saveRDS(rq_shape, (file.path(path_outputs_national, purpose, geography, "rq_all.Rds")) , version = 2)
 }
 
 # MERGE DESTINATION DATA TO DESTINATIONS FILE
@@ -135,7 +135,7 @@ if(purpose == "school") {
   d_shape <- d_shape[sel_zone,]  
   d_shape@data <- data.frame(urn = d_shape$urn) 
   d_shape@data <- left_join(d_shape@data, d_all_attributes, by="urn")
-  saveRDS(d_shape, file.path(path_outputs_national, purpose, geography, "d_all.Rds"))
+  saveRDS(d_shape, file.path(path_outputs_national, purpose, geography, "d_all.Rds") , version = 2)
   geojson_write(d_shape, file = file.path(path_outputs_national, purpose, geography, "d_all.geojson"))
 }
 
@@ -146,21 +146,21 @@ if(purpose == "school") {
   z_shape_private <- z_shape_private[sel_zone,]  
   z_shape_private@data <- data.frame(geo_code = z_shape_private$geo_code) 
   z_shape_private@data <- left_join(z_shape_private@data, z_all_attributes_private, by="geo_code")
-  saveRDS(z_shape_private, file.path(path_outputs_national, purpose_private, geography, "z_all.Rds"))
+  saveRDS(z_shape_private, file.path(path_outputs_national, purpose_private, geography, "z_all.Rds") , version = 2)
 
   print("private_d")
   print(summary({sel_zone <- d_shape_private$urn %in% d_all_attributes_private$urn})) # 206 false = schools excluded from study pop. 
   d_shape_private <- d_shape_private[sel_zone,]  
   d_shape_private@data <- data.frame(urn = d_shape_private$urn) 
   d_shape_private@data <- left_join(d_shape_private@data, d_all_attributes_private, by="urn")
-  saveRDS(d_shape_private, file.path(path_outputs_national, purpose_private, geography, "d_all.Rds"))
+  saveRDS(d_shape_private, file.path(path_outputs_national, purpose_private, geography, "d_all.Rds") , version = 2)
 }
 
 # MERGE LA DATA TO LA GEO FILE [SAME REGARDLESS OF MSOA/LSOA] 
 summary({sel_lad <- (lad$lad11cd %in% lad_attributes$lad11cd)}) # 22 false schools = Wales
 lad <- lad[sel_lad,]  
 lad@data <- left_join(lad@data, lad_attributes, by = "lad11cd")
-saveRDS(lad, (file.path(path_outputs_national, purpose, "lad.Rds")))
+saveRDS(lad, (file.path(path_outputs_national, purpose, "lad.Rds")) , version = 2)
 geojson_write(lad, file = file.path(path_outputs_national, purpose, "lad.geojson"))
 
 # MERGE REGION DATA TO REGION GEO FILE [SAME REGARDLESS OF MSOA/LSOA] 

diff --git a/06b_national_rnet.R b/06b_national_rnet.R
@@ -40,7 +40,7 @@ summary(rf_all) # looks good:
 # Mean   :  2.641                          
 # 3rd Qu.:  2.000                          
 # Max.   :361.000
-# saveRDS(rf_all, "../pct-largefiles/rf_all.Rds")
+# saveRDS(rf_all, "../pct-largefiles/rf_all.Rds", version = 2)
 
 # read-in cleaned file
 rf_all = readRDS("../pct-largefiles/rf_all.Rds")
@@ -89,23 +89,23 @@ for(r in rs) {
   filename_full = paste0("../pct-outputs-regional-R/commute/lsoa/", r, "/rnet_full.Rds")
   filename_rnet = paste0("../pct-outputs-regional-R/commute/lsoa/", r, "/rnet.Rds")
 
-  saveRDS(rnet_intern, filename_intern)
-  saveRDS(rnet_extern, filename_extern)
+  saveRDS(rnet_intern, filename_intern, version = 2)
+  saveRDS(rnet_extern, filename_extern, version = 2)
 
   message("Generating combined rnet")
   rnet_combined = rbind(rnet_intern, rnet_extern)
   rnet = overline2(rnet_combined, attrib = scenarios)
   # plot(rnet)
 
-  saveRDS(rnet, paste0("../pct-outputs-regional-R/commute/lsoa/", r, "/rnet_sf.Rds"))
+  saveRDS(rnet, paste0("../pct-outputs-regional-R/commute/lsoa/", r, "/rnet_sf.Rds"), version = 2)
   rnet_full = cbind(local_id = 1:nrow(rnet), rnet)
-  saveRDS(as(rnet_full, "Spatial"), filename_full)
+  saveRDS(as(rnet_full, "Spatial"), filename_full, version = 2)
 
   rnet_subset = rnet_full[tail(order(rnet_full$dutch_slc), max_nrow_net), ]
   dutch_slc_min = round(min(rnet_subset$dutch_slc / 10)) * 10
   rnet = rnet_full[rnet_full$dutch_slc >= dutch_slc_min, ]
   plot(rnet[rnet$dutch_slc > 100, ]) # test it works
-  saveRDS(as(rnet, "Spatial"), filename_rnet)
+  saveRDS(as(rnet, "Spatial"), filename_rnet, version = 2)
 
   # add log data
   log_data$dutch_slc_min[i] = dutch_slc_min
@@ -149,7 +149,7 @@ rnet_sample = rnet_nat[sample(nrow(rnet_nat), size = 1000), ]
 plot(rnet_sample, lwd = rnet_sample$govtarget_slc / mean(rnet_sample$bicycle))
 mapview::mapview(rnet_sample, lwd = rnet_sample$govtarget_slc / mean(rnet_sample$bicycle) * 10)
 filename_rnet_nat = "../pct-outputs-national/commute/lsoa/rnet_all.Rds"
-saveRDS(rnet_nat, filename_rnet_nat)
+saveRDS(rnet_nat, filename_rnet_nat, version = 2)
 rnet_nat_sf = sf::st_as_sf(rnet_nat)
 sf::st_write(rnet_nat_sf, "rnet_all.gpkg")
 piggyback::pb_upload("rnet_all.gpkg")

diff --git a/07b.1_commute_build_region.R b/07b.1_commute_build_region.R
@@ -107,18 +107,18 @@ if (region_build_param$to_rebuild_rnet=="1") {
 # SAVE OBJECTS
 write_csv(od_attributes, file.path(path_outputs_regional_notR, purpose, geography, region, "od_attributes.csv"))
 write_csv(z@data, file.path(path_outputs_regional_notR, purpose, geography, region, "z_attributes.csv"))
-saveRDS(z, (file.path(path_outputs_regional_R, purpose, geography, region, "z.Rds")))
+saveRDS(z, (file.path(path_outputs_regional_R, purpose, geography, region, "z.Rds")) , version = 2)
 geojson_write(z, file = file.path(path_outputs_regional_notR, purpose, geography, region, "z.geojson"))
-saveRDS(c, (file.path(path_outputs_regional_R, purpose, geography, region, "c.Rds")))
+saveRDS(c, (file.path(path_outputs_regional_R, purpose, geography, region, "c.Rds")) , version = 2)
 geojson_write(c, file = file.path(path_outputs_regional_notR, purpose, geography, region, "c.geojson"))
-saveRDS(l, (file.path(path_outputs_regional_R, purpose, geography, region, "l.Rds")))
+saveRDS(l, (file.path(path_outputs_regional_R, purpose, geography, region, "l.Rds")) , version = 2)
 geojson_write(l, file = file.path(path_outputs_regional_notR, purpose, geography, region, "l.geojson"))
-saveRDS(rf, (file.path(path_outputs_regional_R, purpose, geography, region, "rf.Rds")))
+saveRDS(rf, (file.path(path_outputs_regional_R, purpose, geography, region, "rf.Rds")) , version = 2)
 geojson_write(rf, file = file.path(path_outputs_regional_notR, purpose, geography, region, "rf.geojson"))
-saveRDS(rq, (file.path(path_outputs_regional_R, purpose, geography, region, "rq.Rds")))
+saveRDS(rq, (file.path(path_outputs_regional_R, purpose, geography, region, "rq.Rds")) , version = 2)
 geojson_write(rq, file = file.path(path_outputs_regional_notR, purpose, geography, region, "rq.geojson"))
 if (region_build_param$to_rebuild_rnet=="1") {
-  saveRDS(rnet, (file.path(path_outputs_regional_R, purpose, geography, region, "rnet.Rds")))
+  saveRDS(rnet, (file.path(path_outputs_regional_R, purpose, geography, region, "rnet.Rds")) , version = 2)
   geojson_write(rnet, file = file.path(path_outputs_regional_notR, purpose, geography, region, "rnet.geojson"))
 }