diff --git a/.github/workflows/run-dbt-prod.yml b/.github/workflows/run-dbt-prod.yml new file mode 100644 index 000000000..8fc42d164 --- /dev/null +++ b/.github/workflows/run-dbt-prod.yml @@ -0,0 +1,38 @@ +# .github/workflows/run-dbt-prod.yml +name: run_dbt_prod + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + # schedule: + # - cron: '0 8 * * *' + +env: + DBT_PROFILES_DIR: ./nyc_parking_violations + DBT_PROJECT_DIR: ./nyc_parking_violations + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Run dbt Prod + run: | + dbt debug + dbt compile --target prod + dbt run --target prod + - name: Test dbt Prod + run: | + dbt test --target prod \ No newline at end of file diff --git a/NOTICE b/NOTICE index 547595f90..4f2d213b2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,11 +1,17 @@ -Copyright 2022 LinkedIn Corporation +Copyright 2024 LinkedIn Corporation All Rights Reserved. Licensed under the LinkedIn Learning Exercise File License (the "License"). See LICENSE in the project root for license information. ATTRIBUTIONS: -[PLEASE PROVIDE ATTRIBUTIONS OR DELETE THIS AND THE ABOVE LINE “ATTRIBUTIONS”] + +Pandas +https://github.com/pandas-dev/pandas +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +Copyright (c) 2011-2021, Open source contributors. +License: BSD 3-Clause +https://opensource.org/licenses/BSD-3-Clause Please note, this project may automatically load third party code from external repositories (for example, NPM modules, Composer packages, or other dependencies). @@ -13,3 +19,38 @@ If so, such third party code may be subject to other license terms than as set forth above. In addition, such third party code may also depend on and load multiple tiers of dependencies. Please review the applicable licenses of the additional dependencies. + +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + +PANDAS +BSD 3-Clause License + +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2011-2023, Open source contributors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 422e0932c..06667c942 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# COURSENAME -This is the repository for the LinkedIn Learning course `course-name`. The full course is available from [LinkedIn Learning][lil-course-url]. +# Data Engineering with Data Build Tool (DBT) +This is the repository for the LinkedIn Learning course `Data Engineering with Data Build Tool (DBT)`. The full course is available from [LinkedIn Learning][lil-course-url]. ![course-name-alt-text][lil-thumbnail-url] +Data Build Tool (dbt) has quickly become an essential tool in many data stacks ranging from startups to big tech for managing data transformations. In this course, data engineer Mark Freeman helps you get started with setting up, running, and managing a dbt project via the open-source offering dbt Core. Learn how to install dbt Core, configure an environment for dbt, create and manage a dbt project, and deploy a dbt project in production. If you’re a data professional tasked with implementing dbt within your organization, recently joined a team utilizing dbt and need to upskill, or just want to learn about dbt to increase your competitiveness within the data job market, check out this course. + _See the readme file in the main branch for updated instructions and information._ ## Instructions This repository has branches for each of the videos in the course. You can use the branch pop up menu in github to switch to a specific branch and take a look at the course at that stage, or you can add `/tree/BRANCH_NAME` to the URL to go to the branch you want to access. @@ -22,15 +24,9 @@ To resolve this issue: Add changes to git using this command: git add . Commit changes using this command: git commit -m "some message" -## Installing -1. To use these exercise files, you must have the following installed: - - [list of requirements for course] -2. Clone this repository into your local machine using the terminal (Mac), CMD (Windows), or a GUI tool like SourceTree. -3. [Course-specific instructions] - [0]: # (Replace these placeholder URLs with actual course URLs) -[lil-course-url]: https://www.linkedin.com/learning/ -[lil-thumbnail-url]: http:// +[lil-course-url]: https://www.linkedin.com/learning/data-engineering-with-dbt +[lil-thumbnail-url]: https://media.licdn.com/dms/image/D560DAQHnrbGtKKAabg/learning-public-crop_675_1200/0/1703199806902?e=2147483647&v=beta&t=Q6jKFTslSvhl3bYTkd0PuIEVnIx26lCa-XWVE5hCmm0 diff --git a/create_dbs.ipynb b/create_dbs.ipynb new file mode 100644 index 000000000..219f0b1d1 --- /dev/null +++ b/create_dbs.ipynb @@ -0,0 +1,66 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "69aa603b", + "metadata": {}, + "outputs": [], + "source": [ + "import duckdb\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f4456138", + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "TRAIN-EPAM-DBT", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db index 7087351ff..83b7ed68f 100644 Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db new file mode 100644 index 000000000..87577aea3 Binary files /dev/null and b/data/prod_nyc_parking_violations.db differ diff --git a/nyc_parking_violations/dbt_project.yml b/nyc_parking_violations/dbt_project.yml index 3793c9aa0..c0cf6dfd2 100644 --- a/nyc_parking_violations/dbt_project.yml +++ b/nyc_parking_violations/dbt_project.yml @@ -34,4 +34,20 @@ models: nyc_parking_violations: # Config indicated by + and applies to all files under models/example/ example: + +materialized: ephemeral + bronze: +materialized: view + silver: + silver_parking_violation_codes: + +materialized: ephemeral + silver_parking_violations: + +materialized: ephemeral + silver_violation_tickets: + +materialized: view + silver_violation_vehicles: + +materialized: view + gold: + +materialized: table +tests: + +store_failures: true + diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql index 217a5f263..4074ab031 100644 --- a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql +++ b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql @@ -4,4 +4,4 @@ SELECT manhattan_96th_st_below, all_other_areas FROM - parking_violation_codes + parking_violation_codes \ No newline at end of file diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql index 46525da2a..05d0679ca 100644 --- a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql +++ b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql @@ -18,6 +18,6 @@ SELECT violation_county, violation_legal_code, vehicle_color, - vehicle_year, + vehicle_year FROM - parking_violations_2023 \ No newline at end of file + parking_violations_2023 diff --git a/nyc_parking_violations/models/docs/docs_block.md b/nyc_parking_violations/models/docs/docs_block.md new file mode 100644 index 000000000..cd410941a --- /dev/null +++ b/nyc_parking_violations/models/docs/docs_block.md @@ -0,0 +1,107 @@ +{% docs violation_code %} +Code representing the specific parking violation. +{% enddocs %} + +{% docs definition %} +Description of the violation for a respective code. +{% enddocs %} + +{% docs manhattan_96th_st_below %} +The fee in $USD for a violation on or below Manhattan 96th Street. +{% enddocs %} + +{% docs all_other_areas %} +The fee in $USD for a violation not on or below Manhattan 96th Street. +{% enddocs %} + +{% docs summons_number %} +Unique identifier for each summons issued for a parking violation. +{% enddocs %} + +{% docs registration_state %} +The state where the vehicle is registered. +{% enddocs %} + +{% docs plate_type %} +The type of license plate. +{% enddocs %} + +{% docs issue_date %} +The date when the summons was issued. +{% enddocs %} + +{% docs vehicle_body_type %} +The body type of the vehicle involved in the violation. +{% enddocs %} + +{% docs vehicle_make %} +The make or brand of the vehicle. +{% enddocs %} + +{% docs issuing_agency %} +The agency that issued the summons. +{% enddocs %} + +{% docs vehicle_expiration_date %} +The date when the vehicle's registration expires. +{% enddocs %} + +{% docs violation_location %} +General location where the violation occurred. +{% enddocs %} + +{% docs violation_precinct %} +Precinct where the violation was identified. +{% enddocs %} + +{% docs issuer_precinct %} +Precinct of the officer or official who issued the summons. +{% enddocs %} + +{% docs issuer_code %} +Unique code identifying the issuer. +{% enddocs %} + +{% docs issuer_command %} +Command or unit of the issuer. +{% enddocs %} + +{% docs issuer_squad %} +Squad detail for the issuer. +{% enddocs %} + +{% docs violation_time %} +Time when the violation occurred. +{% enddocs %} + +{% docs violation_county %} +County where the violation took place. +{% enddocs %} + +{% docs violation_legal_code %} +Legal code associated with the violation. +{% enddocs %} + +{% docs vehicle_color %} +Color of the vehicle involved in the violation. +{% enddocs %} + +{% docs vehicle_year %} +Manufacturing year of the vehicle. +{% enddocs %} + +{% docs fee_usd %} +The fee charged for a parking violation, specified in USD. This fee varies depending on the location of the violation. +{% enddocs %} + +{% docs is_manhattan_96th_st_below %} +A boolean value indicating whether the violation occurred in Manhattan on or below 96th Street. +{% enddocs %} + +{% docs ticket_count %} +The total number of tickets issued for a specific violation code. +{% enddocs %} + +{% docs total_revenue_usd %} +The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD. +{% enddocs %} diff --git a/nyc_parking_violations/models/docs/schema.yml b/nyc_parking_violations/models/docs/schema.yml new file mode 100644 index 000000000..c1dfcb9fc --- /dev/null +++ b/nyc_parking_violations/models/docs/schema.yml @@ -0,0 +1,188 @@ +models: + - name: bronze_parking_violation_codes + description: Raw data representing the violation codes and their fees. + columns: + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: definition + description: '{{ doc("definition") }}' + - name: manhattan_96th_st_below + description: '{{ doc("manhattan_96th_st_below") }}' + - name: all_other_areas + description: '{{ doc("all_other_areas") }}' + + - name: bronze_parking_violations + description: Raw data related to parking violations in 2023, encompassing various details about each violation. + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + tests: + - unique + - not_null + - generic_not_null + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: plate_type + description: '{{ doc("plate_type") }}' + - name: issue_date + description: '{{ doc("issue_date") }}' + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: vehicle_body_type + description: '{{ doc("vehicle_body_type") }}' + - name: vehicle_make + description: '{{ doc("vehicle_make") }}' + - name: issuing_agency + description: '{{ doc("issuing_agency") }}' + - name: vehicle_expiration_date + description: '{{ doc("vehicle_expiration_date") }}' + - name: violation_location + description: '{{ doc("violation_location") }}' + - name: violation_precinct + description: '{{ doc("violation_precinct") }}' + - name: issuer_precinct + description: '{{ doc("issuer_precinct") }}' + - name: issuer_code + description: '{{ doc("issuer_code") }}' + - name: issuer_command + description: '{{ doc("issuer_command") }}' + - name: issuer_squad + description: '{{ doc("issuer_squad") }}' + - name: violation_time + description: '{{ doc("violation_time") }}' + - name: violation_county + description: '{{ doc("violation_county") }}' + - name: violation_legal_code + description: '{{ doc("violation_legal_code") }}' + - name: vehicle_color + description: '{{ doc("vehicle_color") }}' + - name: vehicle_year + description: '{{ doc("vehicle_year") }}' + + - name: silver_parking_violation_codes + description: "This model unifies violation codes, providing a comprehensive view of violations, indicating whether they occurred on/below 96th St in Manhattan or in other areas, along with the respective fees in USD." + columns: + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: definition + description: '{{ doc("definition") }}' + - name: is_manhattan_96th_st_below + description: '{{ doc("is_manhattan_96th_st_below") }}' + - name: fee_usd + description: '{{ doc("fee_usd") }}' + + - name: silver_parking_violations + description: "Enhanced view of parking violations, enriched with details and specific indicators such as the flag for violations in Manhattan on or below 96th Street." + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: plate_type + description: '{{ doc("plate_type") }}' + - name: issue_date + description: '{{ doc("issue_date") }}' + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: vehicle_body_type + description: '{{ doc("vehicle_body_type") }}' + - name: vehicle_make + description: '{{ doc("vehicle_make") }}' + - name: issuing_agency + description: '{{ doc("issuing_agency") }}' + - name: vehicle_expiration_date + description: '{{ doc("vehicle_expiration_date") }}' + - name: violation_location + description: '{{ doc("violation_location") }}' + - name: violation_precinct + description: '{{ doc("violation_precinct") }}' + - name: issuer_precinct + description: '{{ doc("issuer_precinct") }}' + - name: issuer_code + description: '{{ doc("issuer_code") }}' + - name: issuer_command + description: '{{ doc("issuer_command") }}' + - name: issuer_squad + description: '{{ doc("issuer_squad") }}' + - name: violation_time + description: '{{ doc("violation_time") }}' + - name: violation_county + description: '{{ doc("violation_county") }}' + - name: violation_legal_code + description: '{{ doc("violation_legal_code") }}' + - name: vehicle_color + description: '{{ doc("vehicle_color") }}' + - name: vehicle_year + description: '{{ doc("vehicle_year") }}' + - name: is_manhattan_96th_st_below + description: '{{ doc("is_manhattan_96th_st_below") }}' + + - name: silver_violation_tickets + description: "Consolidated information on parking violations, enriched with associated fee details." + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + - name: issue_date + description: '{{ doc("issue_date") }}' + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: is_manhattan_96th_st_below + description: '{{ doc("is_manhattan_96th_st_below") }}' + - name: issuing_agency + description: '{{ doc("issuing_agency") }}' + - name: violation_location + description: '{{ doc("violation_location") }}' + - name: violation_precinct + description: '{{ doc("violation_precinct") }}' + - name: issuer_precinct + description: '{{ doc("issuer_precinct") }}' + - name: issuer_code + description: '{{ doc("issuer_code") }}' + - name: issuer_command + description: '{{ doc("issuer_command") }}' + - name: issuer_squad + description: '{{ doc("issuer_squad") }}' + - name: violation_time + description: '{{ doc("violation_time") }}' + - name: violation_county + description: '{{ doc("violation_county") }}' + - name: violation_legal_code + description: '{{ doc("violation_legal_code") }}' + + - name: silver_violation_vehicles + description: "Details of the vehicles involved in parking violations." + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: plate_type + description: '{{ doc("plate_type") }}' + - name: vehicle_body_type + description: '{{ doc("vehicle_body_type") }}' + - name: vehicle_make + description: '{{ doc("vehicle_make") }}' + - name: vehicle_expiration_date + description: '{{ doc("vehicle_expiration_date") }}' + - name: vehicle_color + description: '{{ doc("vehicle_color") }}' + - name: vehicle_year + description: '{{ doc("vehicle_year") }}' + + - name: gold_ticket_metrics + description: "Aggregated metrics representing the total tickets and revenue by violation code." + columns: + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: ticket_count + description: '{{ doc("ticket_count") }}' + - name: total_revenue_usd + description: '{{ doc("total_revenue_usd") }}' + + - name: gold_vehicles_metrics + description: "Aggregated metrics detailing the number of tickets per vehicle, identified by the plate ID." + columns: + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: ticket_count + description: '{{ doc("ticket_count") }}' diff --git a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql index 861de4d61..19cae7fcb 100644 --- a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql +++ b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql @@ -2,7 +2,7 @@ SELECT violation_code, COUNT(summons_number) AS ticket_count, SUM(fee_usd) AS total_revenue_usd -FROM +FROM {{ref('silver_violation_tickets')}} GROUP BY violation_code diff --git a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql index af796bfa9..9b2d280f2 100644 --- a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql +++ b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql @@ -1,6 +1,6 @@ SELECT registration_state, - COUNT(summons_number) AS ticket_count, + COUNT(summons_number) AS ticket_count FROM {{ref('silver_violation_vehicles')}} WHERE diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index 0384bb621..4df86cdbc 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -1,6 +1,12 @@ nyc_parking_violations: outputs: - dev: - type: duckdb - path: '../data/nyc_parking_violations.db' + dev: + type: duckdb + path: '../data/nyc_parking_violations.db' + prod: + type: duckdb + # note that path is slightly different as GitHub action + # start in the root directory and not in the + # nyc_parking_violation_directory + path: '../data/prod_nyc_parking_violations.db' target: dev diff --git a/nyc_parking_violations/tests/generic/generic_not_null.sql b/nyc_parking_violations/tests/generic/generic_not_null.sql new file mode 100644 index 000000000..07e602d5c --- /dev/null +++ b/nyc_parking_violations/tests/generic/generic_not_null.sql @@ -0,0 +1,7 @@ +{% test generic_not_null(model, column_name) %} + + select * + from {{ model }} + where {{ column_name }} is null + +{% endtest %} \ No newline at end of file diff --git a/nyc_parking_violations/tests/violation_codes_revenue.sql b/nyc_parking_violations/tests/violation_codes_revenue.sql new file mode 100644 index 000000000..5ee4022d1 --- /dev/null +++ b/nyc_parking_violations/tests/violation_codes_revenue.sql @@ -0,0 +1,10 @@ +{{ config(severity = 'warn') }} +SELECT + violation_code, + SUM(fee_usd) as total_revenue_usd +FROM + {{ref('silver_parking_violation_codes')}} +GROUP BY + violation_code +HAVING + NOT(total_revenue_usd >= 1) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 557cf8617..36e1da1dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ dbt-core==1.6.1 dbt-duckdb==1.6.0 duckdb==0.9.0 pandas==2.1.1 +numpy==1.26.4 diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index 00838e7f1..d5e61cd08 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,95 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
violation_codetotal_revenue_usd
0410.0
\n", + "
" + ], + "text/plain": [ + " violation_code total_revenue_usd\n", + "0 41 0.0" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sql_query = '''\n", + "select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n", + "\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " display(con.sql(sql_query).df())" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "sql_query_import_2 = ''' \n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -50,42 +138,26 @@ " \n", " \n", " 2\n", - " first_model\n", - " \n", - " \n", - " 3\n", " gold_ticket_metrics\n", " \n", " \n", - " 4\n", + " 3\n", " gold_vehicles_metrics\n", " \n", " \n", - " 5\n", + " 4\n", " parking_violation_codes\n", " \n", " \n", - " 6\n", + " 5\n", " parking_violations_2023\n", " \n", " \n", - " 7\n", - " ref_model\n", - " \n", - " \n", - " 8\n", - " silver_parking_violation_codes\n", - " \n", - " \n", - " 9\n", - " silver_parking_violations\n", - " \n", - " \n", - " 10\n", + " 6\n", " silver_violation_tickets\n", " \n", " \n", - " 11\n", + " 7\n", " silver_violation_vehicles\n", " \n", " \n", @@ -93,19 +165,15 @@ "" ], "text/plain": [ - " name\n", - "0 bronze_parking_violation_codes\n", - "1 bronze_parking_violations\n", - "2 first_model\n", - "3 gold_ticket_metrics\n", - "4 gold_vehicles_metrics\n", - "5 parking_violation_codes\n", - "6 parking_violations_2023\n", - "7 ref_model\n", - "8 silver_parking_violation_codes\n", - "9 silver_parking_violations\n", - "10 silver_violation_tickets\n", - "11 silver_violation_vehicles" + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 gold_ticket_metrics\n", + "3 gold_vehicles_metrics\n", + "4 parking_violation_codes\n", + "5 parking_violations_2023\n", + "6 silver_violation_tickets\n", + "7 silver_violation_vehicles" ] }, "metadata": {}, @@ -115,23 +183,17 @@ "source": [ "sql_query = '''\n", "show tables\n", + "\n", "'''\n", "\n", - "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "TRAIN-EPAM-DBT", "language": "python", "name": "python3" }, @@ -145,14 +207,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.4" }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" - } - } + "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2