Standardize the last step: disconnect & close Docker

Author: smithjd

050-docker-setup-postgres-with-adventureworks.Rmd

@@ -141,7 +141,7 @@ dbDisconnect(con)
 
 ```
 
-## Cleaning up
+## Cleaning up: diconnect from the database and stop Docker
 
 Always have R disconnect from the database when you're done.
 ```{r}

060-secure-and-use-dbms-credentials-in-r.Rmd

@@ -73,15 +73,9 @@ Once the connection object has been created, you can list all of the tables in o
 dbExecute(con, "set search_path to humanresources, public;") # watch for duplicates!
 dbListTables(con)
 ```
+## Disconnect from the database and stop Docker
 
-## Clean up
-
-Afterwards, always disconnect from the dbms:
 ```{r}
 dbDisconnect(con)
-```
-Tell Docker to stop the `adventureworks` container:
-```{r}
 sp_docker_stop("adventureworks")
 ```
-

065-connect-to-adventureworks-with-dplyr.Rmd

@@ -134,21 +134,9 @@ salesorderheader_table$ops$x$vars
 ```
 `salesorderheader_table` holds information needed to get the data from the 'salesorderheader' table, but `salesorderheader_table` does not hold the data itself. In the following sections, we will examine more closely this relationship between the `salesorderheader_table` object and the data in the database's 'salesorderheader' table.
 
-Disconnect from the database:
-```{r }
-dbDisconnect(con)
+## Disconnect from the database and stop Docker
 
-```
-## Cleaning up
-
-Always have R disconnect from the database when you're done.
 ```{r}
-
 dbDisconnect(con)
-
-```
-
-Stop the `adventureworks` container:
-```{r}
 sp_docker_stop("adventureworks")
 ```

080-elementary-queries.Rmd

@@ -283,12 +283,11 @@ skimr::skim_to_wide(salesorderheader_tibble) #skimr doesn't like certain kinds o
 
 ```
 
-### Close the connection and shut down adventureworks
+## Disconnect from the database and stop Docker
 
-Where you place the `collect` function matters.
 ```{r}
-DBI::dbDisconnect(con)
-sqlpetr::sp_docker_stop("adventureworks")
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
 ```
 
 ## Additional reading

083-exploring-a-single-table.Rmd

@@ -324,15 +324,17 @@ ggplot(data = annual_sales, aes(x = orderdate, y = avg_total_soh_dollars)) +
 
 Look at number of orders by the the average sales per order for the four years:
 ```{r number of orders by the the average sales per order 2}
-annual_sales %>% arrange(orderdate) %>% 
-ggplot(aes(x = avg_total_soh_dollars, y = as.numeric(soh_count))) +
+sales <- annual_sales %>% arrange(orderdate) %>% 
+  ungroup() %>% 
+  mutate(year = year(orderdate), year = as.factor(year)) 
+  
+sales %>% ggplot(aes(x = avg_total_soh_dollars, y = as.numeric(soh_count), color = year)) +
   geom_point() +
-  geom_text(aes(label = orderdate, hjust = .5, vjust = 0)) +
+  geom_text(aes(label = year, hjust = .5, vjust = 0)) +
   geom_path() +
   xlab("Average dollars per order") +
   ylab("Total number of orders") +
-    facet_wrap("onlineorderflag", scales = "free") +
-  scale_y_continuous(labels = scales::dollar_format()) + 
+  facet_wrap("onlineorderflag", scales = "free") +     
   scale_x_continuous(labels = scales::dollar_format()) + 
   ggtitle(paste("Number of Orders by Average Order Amount\n", 
                 min_soh_dt, " - ", max_soh_dt))
@@ -399,6 +401,86 @@ A couple of things jump out from the graph.
 2.  2011 has more variation than 2012 and 2013 and peaks every two months.
 3.  2014 has the most variation and also peaks every two months.  Both the number of sales, 939, and the average sales order size, $52.19 plummet in June 2014.
 
+## define the one-off function
+
+```{r}
+monthly_sales <-  tbl(con, in_schema("sales", "salesorderheader")) %>%
+  select(orderdate, subtotal, onlineorderflag) %>% 
+  # mutate(day = day(as.Date(orderdate))) %>% 
+  mutate(
+    orderdate = as.Date(orderdate),
+    day = day(orderdate)) %>% 
+     show_query() %>% 
+  collect() %>% # From here on we're in R
+    mutate(   
+    correct_orderdate = case_when(
+      # onlineorderflag == FALSE & day == 1 ~ NA,
+      onlineorderflag == FALSE & day == 1L ~ orderdate - 1 ,
+      TRUE ~ orderdate
+  ))
+
+```
+
+Here's the magic:
+
+```{r}
+dbExecute(con,
+"CREATE OR REPLACE FUNCTION so_adj_date(so_date timestamp, ONLINE_ORDER boolean) RETURNS timestamp AS $$
+     BEGIN
+        IF (ONLINE_ORDER) THEN
+            RETURN (SELECT so_date);
+        ELSE
+            RETURN(SELECT CASE WHEN EXTRACT(DAY FROM so_date) = 1
+                               THEN  so_date - '1 day'::interval
+                               ELSE  so_date
+                          END
+                  );
+        END IF;
+ END; $$
+LANGUAGE PLPGSQL;
+")
+```
+
+this is a big lesson!  Using the magic.
+
+```{r}
+monthly_sales <-  tbl(con, in_schema("sales", "salesorderheader")) %>%
+  select(orderdate, subtotal, onlineorderflag) %>% 
+  # mutate(day = day(as.Date(orderdate))) %>% 
+  mutate(
+    # orderdate = as.Date(orderdate),
+    adjusted_date = so_adj_date(orderdate, onlineorderflag),
+    day = day(orderdate)) %>% 
+     show_query() %>% 
+  # collect() %>% # From here on we're in R
+    collect()
+
+```
+
+```{r}
+monthly_sales %>% filter(day==1 & onlineorderflag == FALSE) %>% 
+  count(orderdate) %>% as.data.frame()
+```
+
+```{r}
+mutate(orderdate = date(orderdate), 
+         orderdate = round_date(orderdate, "month"),
+          onlineorderflag = if_else(onlineorderflag == FALSE, 
+                                   "Sales Rep", "Online"),) %>%  # 
+  group_by(orderdate, onlineorderflag) %>%
+  summarize(
+    min_soh_orderdate = min(orderdate, na.rm = TRUE),
+    max_soh_orderdate = max(orderdate, na.rm = TRUE),
+    total_soh_dollars = round(sum(subtotal, na.rm = TRUE), 2),
+    avg_total_soh_dollars = round(mean(subtotal, na.rm = TRUE), 2),
+    soh_count = n()
+  ) 
+
+```
+
+
+
+
 ### Looking at lagged data
 
 ```{r}
@@ -759,11 +841,9 @@ The gather command throws the following warning:
     they will be dropped
 
 
-## Close and clean up
-
+## Disconnect from the database and stop Docker
 
 ```{r}
 dbDisconnect(con)
 sp_docker_stop("adventureworks")
 ```
-

085-sales-forecasting.Rmd

@@ -82,6 +82,20 @@ First, let's look at orders for online and sales representative sales:
 
 ```{r}
 monthly_tsibble %>% autoplot(orders)
+Disconnect from the database:
+```{r }
+dbDisconnect(con)
+
+```
+
+## Cleaning up
+
+Always have R disconnect from the database when you're done.
+```{r}
+
+dbDisconnect(con)
+
+```
 
 ```
 
@@ -103,3 +117,10 @@ monthly_tsibble %>% autoplot(total_revenue / orders)
 ```
 
 For the sales representatives, there's still a month-to-month variation but the revenue per order appears to be bounded both below and above. However, the online revenue per order is decreasing. Note that this decline appears to be in steps between May and June each year; that could mean it's an artifact of the database creation process and not a "natural" phenomenon.
+
+## Disconnect from the database and stop Docker
+
+```{r}
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
+```

090-lazy-evalutation-intro.Rmd

@@ -235,10 +235,11 @@ Q %>% dplyr::show_query()
 
 ```
 Hand-written SQL code to do the same job will probably look a lot nicer and could be more efficient, but functionally `dplyr` does the job.
+## Disconnect from the database and stop Docker
 
-```{r disconnect and close down}
-DBI::dbDisconnect(con)
-sqlpetr::sp_docker_stop("adventureworks")
+```{r}
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
 ```
 
 

095-lazy-evalutation-and-timing.Rmd

@@ -145,12 +145,13 @@ try(sales_person_table %>% collect() %>%
 
 See more examples of lazy execution [here](https://datacarpentry.org/R-ecology-lesson/05-r-and-databases.html).
 
+## Disconnect from the database and stop Docker
+
 ```{r}
-DBI::dbDisconnect(con)
-sqlpetr::sp_docker_stop("adventureworks")
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
 ```
 
-
 ## Other resources
 
 * Benjamin S. Baumer. 2017. A Grammar for Reproducible and Painless Extract-Transform-Load Operations on Medium Data. [https://arxiv.org/abs/1708.07073](https://arxiv.org/abs/1708.07073) 

100-dbi-and-sql.Rmd

@@ -261,7 +261,10 @@ GQ <- dbGetQuery(
 But because `Q` hasn't been executed, we can add to it.  This behavior is the basis for a useful debugging and development process where queries are built up incrementally.
 
 Where you place the `collect` function matters.
+
+## Disconnect from the database and stop Docker
+
 ```{r}
-DBI::dbDisconnect(con)
-sqlpetr::sp_docker_stop("sql-pet")
+dbDisconnect(con)
+sp_docker_stop("sql-pet")
 ```

130-sql-dplyr-joins-data.Rmd

@@ -488,8 +488,9 @@ smy_cust_dupes <- dbGetQuery(con,'select *
 sp_print_df(smy_cust_dupes)
 ```
 -->
+## Disconnect from the database and stop Docker
+
 
-Diconnect from the db:
 ```{r}
 
  dbDisconnect(con)

140-sql-dplyr-joins.Rmd

@@ -928,11 +928,11 @@ The explanation below is from [here](https://stackoverflow.com/questions/4748577
 
 In `by = c("col1" = "col2")`,  `=` is _not_ the equality operator, but an assignment operator (the equality operator in R is `==`). The expression inside `c(...)` creates a _named character vector_ (name: col1 value: col2) that `dplyr` uses for the join. Nowhere do you define the kind of comparison that is made during the join. The comparison is hard-coded in `dplyr`. I don't think `dplyr` supports non-equi joins (yet).
 
-```{r}
-# disconnect from the db
- dbDisconnect(con)
+## Disconnect from the database and stop Docker
 
- sp_docker_stop("sql-pet")
+```{r}
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
 ```
 
 ```{r}

150-sql-joins-exercises.Rmd

@@ -318,3 +318,9 @@ In the next code block, change the function parameter to different table names t
 #dbListTables(con)
 sp_tbl_pk_fk_sql('customer')
 ```
+## Disconnect from the database and stop Docker
+
+```{r}
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
+```

153-sql-joins-exercises.Rmd

@@ -1683,7 +1683,7 @@ cust_id <- 601
 sp_print_df(customer_details_fn_dplyr(cust_id))
 ```
 
-## Diconnect from the db and clean up
+## Diconnect from the database and stop Docker
 ```{r}
 dbDisconnect(con)
 

155-sql-joins-exercises-with-answers.Rmd

@@ -2191,11 +2191,11 @@ compute(smy_customer_details_dplyr,name='smy_compute_exercise',temporary = FALSE
 
 ```
 
+## Disconnect from the database and stop Docker
+
 ```{r}
-# diconnect from the db
 dbDisconnect(con)
-
-sp_docker_stop("sql-pet")
+sp_docker_stop("adventureworks")
 ```
 
 ```{r}

160-anti-join-cost-comparisons.Rmd

@@ -173,3 +173,12 @@ select c.customer_id,count(*) lojs
 print(glue("sql_aj1 loj-null costs=", sql_aj1[1, 1]))
 print(glue("sql_aj3 not exist costs=", sql_aj3[1, 1]))
 ```
+## Cleaning up
+
+Always have R disconnect from the database when you're done and stop the Adventureworks Container
+```{r}
+
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
+
+```

170-sql-quick-start-simple-retrieval.Rmd

@@ -509,3 +509,14 @@ The SEQ column shows the standard order of SQL execution.  One take away for thi
       
   6.  SQL for View table dependencies.
   7.  Add cartesian join exercise.
+
+```
+## Cleaning up
+
+Always have R disconnect from the database when you're done and stop the Adventureworks Container
+```{r}
+
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
+
+```

180-r-query-postgres-metadata.Rmd

@@ -343,3 +343,13 @@ ls()
 ```
 
 
+```
+## Cleaning up
+
+Always have R disconnect from the database when you're done and stop the Adventureworks Container
+```{r}
+
+dbDisconnect(con)
+sp_docker_stop("adventureworks")
+
+```

190-drilling-into-db-environment.Rmd

@@ -393,8 +393,13 @@ create_graph(
 ) %>%
 render_graph()
 ```
+```
+## Cleaning up
 
+Always have R disconnect from the database when you're done and stop the Adventureworks Container
 ```{r}
+
 dbDisconnect(con)
-# system2('docker','stop sql-pet')
+sp_docker_stop("adventureworks")
+
 ```

210-sql-query-steps.Rmd

@@ -75,8 +75,12 @@ dbClearResult(rs)
 # dbRemoveTable(con, "cars")
 dbRemoveTable(con, "mtcars")
 # dbRemoveTable(con, "cust_movies")
+```
+
+
+```{r}
+## Disconnect from the database and stop Docker
 
-# diconnect from the db
 dbDisconnect(con)
 
 sp_docker_stop("sql-pet")