diff --git a/.github/workflows/google-gce-dev.yaml b/.github/workflows/google-gce-dev.yaml new file mode 100644 index 000000000..a8c78b8db --- /dev/null +++ b/.github/workflows/google-gce-dev.yaml @@ -0,0 +1,141 @@ +name: "Deploy to GCE development" + +on: + push: + branches: [ "develop" ] + +env: + GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} + GCP_ARTIFACT_REGISTRY_NAME: ${{ vars.GCP_ARTIFACT_REGISTRY_NAME }} + GCP_ARTIFACT_REGISTRY_LOCATION: ${{ vars.GCP_ARTIFACT_REGISTRY_LOCATION }} + GCP_IMAGE_NAME: ${{ vars.GCP_IMAGE_NAME }} + GCP_VM_IP: ${{ vars.GCP_VM_IP }} + GCP_VM_USER: ${{ vars.GCP_VM_USER }} + + CANON_API: ${{ vars.CANON_API }} + CANON_CMS_CUBES: ${{ vars.CANON_CMS_CUBES }} + CANON_CMS_ENABLE: ${{ vars.CANON_CMS_ENABLE }} + CANON_CMS_FORCE_HTTPS: ${{ vars.CANON_CMS_FORCE_HTTPS }} + CANON_CMS_GENERATOR_TIMEOUT: ${{ vars.CANON_CMS_GENERATOR_TIMEOUT }} + CANON_CMS_LOGGING: ${{ vars.CANON_CMS_LOGGING }} + CANON_CMS_MINIMUM_ROLE: ${{ vars.CANON_CMS_MINIMUM_ROLE }} + CANON_CMS_REQUESTS_PER_SECOND: ${{ vars.CANON_CMS_REQUESTS_PER_SECOND }} + CANON_CONST_CART: ${{ vars.CANON_CONST_CART }} + CANON_CONST_CUBE: ${{ vars.CANON_CONST_CUBE }} + CANON_CONST_TESSERACT: ${{ vars.CANON_CONST_TESSERACT }} + CANON_DB_NAME: ${{ vars.CANON_DB_NAME }} + CANON_DB_USER: ${{ vars.CANON_DB_USER }} + CANON_GEOSERVICE_API: ${{ vars.CANON_GEOSERVICE_API }} + CANON_GOOGLE_ANALYTICS: ${{ vars.CANON_GOOGLE_ANALYTICS }} + CANON_LANGUAGES: ${{ vars.CANON_LANGUAGES }} + CANON_LANGUAGE_DEFAULT: ${{ vars.CANON_LANGUAGE_DEFAULT }} + CANON_LOGICLAYER_CUBE: ${{ vars.CANON_LOGICLAYER_CUBE }} + CANON_LOGICLAYER_SLUGS: ${{ vars.CANON_LOGICLAYER_SLUGS }} + CANON_LOGINS: ${{ vars.CANON_LOGINS }} + GA_KEYFILE: ${{ vars.GA_KEYFILE }} + + +jobs: + build: + environment: development-vm + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Authenticate with Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Build Docker Image + run: |- + gcloud builds submit \ + --quiet \ + --timeout=30m \ + --config=cloudbuild.yml \ + --substitutions=_GCP_PROJECT_ID=${{ env.GCP_PROJECT_ID }},_GCP_ARTIFACT_REGISTRY_NAME=${{ env.GCP_ARTIFACT_REGISTRY_NAME }},_GCP_ARTIFACT_REGISTRY_LOCATION=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }},_GCP_IMAGE_NAME=${{ env.GCP_IMAGE_NAME }},_GCP_IMAGE_TAG=${{ github.sha }},_GCP_IMAGE_ENVIRONMENT=${{ env.GCP_IMAGE_NAME }} + + deploy: + needs: build + environment: development-vm + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Authenticate with Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Deploy to Compute Engine + run: | + SSH_DIR=~/.ssh + mkdir -p $SSH_DIR + echo "${{ secrets.GCP_SSH_PRIVATE_KEY }}" > $SSH_DIR/id_rsa + chmod 600 $SSH_DIR/id_rsa + ssh-keyscan -H ${{ env.GCP_VM_IP }} >> $SSH_DIR/known_hosts + + # Define remote path + REMOTE_PATH="/home/${{ env.GCP_VM_USER }}/${{ env.GCP_ARTIFACT_REGISTRY_NAME }}-${{ env.GCP_IMAGE_NAME }}" + + # Create remote directory and transfer files in one SSH session + echo "Creating remote path" + ssh -i $SSH_DIR/id_rsa ${{ env.GCP_VM_USER }}@${{ env.GCP_VM_IP }} "mkdir -p $REMOTE_PATH" + echo "Sending files" + scp -i $SSH_DIR/id_rsa compose.yaml deploy_to_vm.sh ${{ env.GCP_VM_USER }}@${{ env.GCP_VM_IP }}:$REMOTE_PATH + + ssh -i $SSH_DIR/id_rsa ${{ env.GCP_VM_USER }}@${{ env.GCP_VM_IP }} 'bash -s' << 'ENDSSH' + + # Go to working directory + echo "Going to working directory" + cd "/home/${{ env.GCP_VM_USER }}/${{ env.GCP_ARTIFACT_REGISTRY_NAME }}-${{ env.GCP_IMAGE_NAME }}" + + # Create .env.gcp file + echo "Creating .env.gcp file" + { + echo "GCP_IMAGE_TAG=${{ github.sha }}" + echo "GCP_PROJECT_ID=${{ env.GCP_PROJECT_ID }}" + echo "GCP_IMAGE_NAME=${{ env.GCP_IMAGE_NAME }}" + echo "GCP_ARTIFACT_REGISTRY_NAME=${{ env.GCP_ARTIFACT_REGISTRY_NAME }}" + echo "GCP_ARTIFACT_REGISTRY_LOCATION=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }}" + echo "GCP_VM_USER=${{ env.GCP_VM_USER }}" + echo "CANON_DB_HOST=${{ secrets.CANON_DB_HOST}}" + echo "CANON_DB_PW=${{ secrets.CANON_DB_PW}}" + echo "CANON_HOTJAR=${{ secrets.CANON_HOTJAR}}" + echo "CANON_API=${{ env.CANON_API }}" + echo "CANON_CMS_CUBES=${{ env.CANON_CMS_CUBES }}" + echo "CANON_CMS_ENABLE=${{ env.CANON_CMS_ENABLE }}" + echo "CANON_CMS_FORCE_HTTPS=${{ env.CANON_CMS_FORCE_HTTPS }}" + echo "CANON_CMS_GENERATOR_TIMEOUT=${{ env.CANON_CMS_GENERATOR_TIMEOUT }}" + echo "CANON_CMS_LOGGING=${{ env.CANON_CMS_LOGGING }}" + echo "CANON_CMS_MINIMUM_ROLE=${{ env.CANON_CMS_MINIMUM_ROLE }}" + echo "CANON_CMS_REQUESTS_PER_SECOND=${{ env.CANON_CMS_REQUESTS_PER_SECOND }}" + echo "CANON_CONST_CART=${{ env.CANON_CONST_CART }}" + echo "CANON_CONST_CUBE=${{ env.CANON_CONST_CUBE }}" + echo "CANON_CONST_TESSERACT=${{ env.CANON_CONST_TESSERACT }}" + echo "CANON_DB_NAME=${{ env.CANON_DB_NAME }}" + echo "CANON_DB_USER=${{ env.CANON_DB_USER }}" + echo "CANON_GEOSERVICE_API=${{ env.CANON_GEOSERVICE_API }}" + echo "CANON_GOOGLE_ANALYTICS=${{ env.CANON_GOOGLE_ANALYTICS }}" + echo "CANON_LANGUAGES=${{ env.CANON_LANGUAGES }}" + echo "CANON_LANGUAGE_DEFAULT=${{ env.CANON_LANGUAGE_DEFAULT }}" + echo "CANON_LOGICLAYER_CUBE=${{ env.CANON_LOGICLAYER_CUBE }}" + echo "CANON_LOGICLAYER_SLUGS=${{ env.CANON_LOGICLAYER_SLUGS }}" + echo "CANON_LOGINS=${{ env.CANON_LOGINS }}" + echo "GA_KEYFILE=${{ env.GA_KEYFILE }}" + } > .env.gcp + + echo "Adding Google Analytics credentials to ./google directory" + mkdir -p ./google + + cat << EOF > ./google/googleAnalyticsKey.json + ${{ secrets.GA_KEYFILE }} + EOF + + bash ./deploy_to_vm.sh + + ENDSSH diff --git a/.github/workflows/google-registry-gke-dev.yaml b/.github/workflows/google-registry-gke-dev.yaml index 060cd70c1..2a6ade1bc 100644 --- a/.github/workflows/google-registry-gke-dev.yaml +++ b/.github/workflows/google-registry-gke-dev.yaml @@ -61,7 +61,7 @@ name: "[GCP][DEV] Build NextJS to Registry and Deploy via Helm" on: push: - branches: [ "develop" ] + branches: [ "disabled" ] paths: - .github/workflows/google-registry-gke-dev.yaml - helm/templates diff --git a/api/crosswalks.js b/api/crosswalks.js index ef99f6b06..71520fc02 100644 --- a/api/crosswalks.js +++ b/api/crosswalks.js @@ -149,7 +149,6 @@ module.exports = function(app) { app.get("/api/:slug/similar/:urlId", async(req, res) => { const {limit, slug, urlId} = req.params; - const meta = await db.profile_meta.findOne({where: {slug}}).catch(() => false); if (!meta) res.json({error: "Not a valid profile type"}); @@ -171,7 +170,7 @@ module.exports = function(app) { if (id === "01000US") { const states = await axios - .get(`${CANON_API}/api/data?drilldowns=State&measure=Household%20Income%20by%20Race&Year=latest&order=Household%20Income%20by%20Race`) + .get(`${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=acs_ygr_median_household_income_race_5&drilldowns=State&locale=en&measures=Household+Income+by+Race&time=Year.latest&sort=Household+Income+by+Race.desc`) .then(resp => { const arr = resp.data.data; const l = Math.ceil(parseFloat(limit || 6) / 2); @@ -181,7 +180,7 @@ module.exports = function(app) { attrs = states.length ? await db.search .findAll({ - where: {id: states.map(d => d["ID State"]), dimension}, + where: {id: states.map(d => d["State ID"]), dimension}, include: [{association: "content"}] }) .catch(() => []) : []; @@ -258,7 +257,6 @@ module.exports = function(app) { } } else { - const parents = await axios.get(`${CANON_API}/api/parents/${slug}/${id}`) .then(resp => resp.data) .catch(() => []); @@ -276,10 +274,9 @@ module.exports = function(app) { "CIP": "Completions", "NAPCS": "Obligation Amount" }; - const neighbors = measures[dimension] ? await axios.get(`${CANON_API}/api/neighbors?dimension=${dimension}&id=${id}&measure=${measures[dimension]}`) - .then(resp => resp.data.data.map(d => d[`ID ${hierarchy}`])) + const neighbors = measures[dimension] ? await axios.get(`${CANON_API}/api/neighbors?dimension=${dimension}&id=${id}&measure=${measures[dimension]}${hierarchy ? `&hierarchy=${hierarchy}` : ""}`) + .then(resp => resp.data.data.map(d => d[`${hierarchy} ID`])) .catch(() => []) : []; - const neighborAttrs = neighbors.length ? await db.search .findAll({ where: {id: neighbors.filter(d => d !== id).map(String), dimension, hierarchy}, @@ -303,11 +300,8 @@ module.exports = function(app) { } return row; }); - res.json(retArray); - } - } }); @@ -408,22 +402,40 @@ module.exports = function(app) { * To handle the sentence: "The highest paying jobs for people who hold a degree in one of the * 5 most specialized majors at University." */ - app.get("/api/university/highestWageLookup/:id", async(req, res) => { + app.get("/api/university/highestWageLookup/:id/:hierarchy", async(req, res) => { + + const ipedsPumsFilterTesseract = d => ![21, 29, 32, 33, 34, 35, 36, 37, 48, 53, 60].includes(d["CIP2 ID"]); + const {id} = req.params; + const {hierarchy} = req.params; + + const latestYearUrl = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=ipeds_completions&drilldowns=Year&include=${hierarchy}:${id}&locale=en&measures=Completions&time=Year.latest` + const latestYear = await axios.get(latestYearUrl) + .then(resp => resp.data.data[0]["Year"]); + + const cipURL = `${CANON_CONST_TESSERACT}complexity/rca_historical.jsonrecords?cube=ipeds_completions&location=${hierarchy}&activity=CIP2&measure=Completions&time=Year&filter=${hierarchy}:${id}&cuts=Year:${latestYear}` - const cipURL = `${CANON_API}/api/data?University=${id}&measures=Completions,yuc%20RCA&year=latest&drilldowns=CIP2&order=yuc%20RCA&sort=desc`; const CIP2 = await axios.get(cipURL) - .then(resp => resp.data.data.filter(ipedsPumsFilter).slice(0, 5).map(d => d["ID CIP2"]).join()); + .then(resp => resp.data.data + .sort((a, b) => b["Completions RCA"] - a["Completions RCA"]) + .filter(ipedsPumsFilterTesseract) + .slice(0, 5) + .map(d => `${d["CIP2 ID"]}`.padStart(2, '0')).join()); + + const logicUrl = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=pums_5&drilldowns=Year,CIP2,Detailed+Occupation&include=Workforce+Status:true;Employment+Time+Status:1;CIP2:${CIP2}&locale=en&measures=Record+Count,Average+Wage&time=Year.latest&filters=Record+Count.gte.5&sort=Average+Wage.desc` - const logicUrl = `${CANON_API}/api/data?measures=Average%20Wage,Record%20Count&year=latest&drilldowns=CIP2,Detailed%20Occupation&order=Average%20Wage&sort=desc&Workforce%20Status=true&Employment%20Time%20Status=1&Record%20Count%3E=5&CIP2=${CIP2}`; const wageList = await axios.get(logicUrl) - .then(resp => resp.data.data); + .then(resp => resp.data.data) + + wageList.sort((a, b) => b["Average Wage"] - a["Average Wage"]); const dedupedWages = []; wageList.forEach(d => { if (dedupedWages.length < 5 && !dedupedWages.find(w => w["Detailed Occupation"] === d["Detailed Occupation"])) dedupedWages.push(d); }); + + dedupedWages.sort((a, b) => b["Average Wage"] - a["Average Wage"]); res.json({data: dedupedWages.slice(0, 10)}); }); @@ -432,23 +444,39 @@ module.exports = function(app) { * To handle the sentence: "The most common industries for people who hold a degree in one * of the 5 most specialized majors at University." */ - app.get("/api/university/commonIndustryLookup/:id", async(req, res) => { + app.get("/api/university/commonIndustryLookup/:id/:hierarchy", async(req, res) => { + + const ipedsPumsFilterTesseract = d => ![21, 29, 32, 33, 34, 35, 36, 37, 48, 53, 60].includes(d["CIP2 ID"]); + const {id} = req.params; + const {hierarchy} = req.params; + + const latestYearUrl = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=ipeds_completions&drilldowns=Year&include=${hierarchy}:${id}&locale=en&measures=Completions&time=Year.latest` + const latestYear = await axios.get(latestYearUrl) + .then(resp => resp.data.data[0]["Year"]); + + const cipURL = `${CANON_CONST_TESSERACT}complexity/rca_historical.jsonrecords?cube=ipeds_completions&location=${hierarchy}&activity=CIP2&measure=Completions&time=Year&filter=${hierarchy}:${id}&cuts=Year:${latestYear}` - const cipURL = `${CANON_API}/api/data?University=${id}&measures=Completions,yuc%20RCA&year=latest&drilldowns=CIP2&order=yuc%20RCA&sort=desc`; const CIP2 = await axios.get(cipURL) - .then(resp => resp.data.data.filter(ipedsPumsFilter).slice(0, 5).map(d => d["ID CIP2"]).join()); + .then(resp => resp.data.data + .sort((a, b) => b["Completions RCA"] - a["Completions RCA"]) + .filter(ipedsPumsFilterTesseract) + .slice(0, 5) + .map(d => `${d["CIP2 ID"]}`.padStart(2, '0')).join()); + + const logicUrl = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=pums_5&drilldowns=Year,CIP2,Industry+Group&include=Workforce+Status:true;Employment+Time+Status:1;CIP2:${CIP2}&locale=en&measures=Record+Count,Total+Population&time=Year.latest&filters=Record+Count.gte.5&sort=Total+Population.desc` - const logicUrl = `${CANON_API}/api/data?measures=Total%20Population,Record%20Count&year=latest&drilldowns=CIP2,Industry%20Group&order=Total%20Population&Workforce%20Status=true&Employment%20Time%20Status=1&sort=desc&Record%20Count>=5&CIP2=${CIP2}`; - const industryList = await axios.get(logicUrl) + let industryList = await axios.get(logicUrl) .then(resp => resp.data.data); const dedupedIndustries = []; // The industryList has duplicates. For example, if a Biology Major enters Biotech, and a separate // Science major enters Biotech, these are listed as separate data points. These must be folded // together under one "Biotech" to create an accurate picture of "industries entered by graduates with X degrees" + industryList = industryList.filter(d => d["Industry Group ID"] !== "") + industryList.forEach(d => { - const thisIndustry = dedupedIndustries.find(j => j["Industry Group"] === d["Industry Group"]); + const thisIndustry = dedupedIndustries.find(j => j["Industry Group"] === d["Industry Group"]); if (thisIndustry) { thisIndustry["Total Population"] += d["Total Population"]; } @@ -456,6 +484,7 @@ module.exports = function(app) { dedupedIndustries.push(d); } }); + dedupedIndustries.sort((a, b) => b["Total Population"] - a["Total Population"]); res.json({data: dedupedIndustries.slice(0, 10)}); @@ -463,13 +492,18 @@ module.exports = function(app) { app.get("/api/parents/:slug/:id", async(req, res) => { - const {slug, id} = req.params; + const {slug} = req.params; + let {id} = req.params; const {loose} = req.query; const meta = await db.profile_meta.findOne({where: {slug}}).catch(() => false); if (!meta) res.json({error: "Not a valid profile type"}); const {dimension} = meta; + if (slug === 'cip' && (`${id}`.length === 1 || `${id}`.length === 3 || `${id}`.length === 5)) { + id = `0${id}`; + } + const attr = await db.search .findOne({ where: {[sequelize.Op.or]: {id, slug: id}, dimension}, @@ -547,7 +581,6 @@ module.exports = function(app) { else { const parents = cache.parents[slug] || {}; const ids = parents[attr.id] || []; - const attrs = ids.length ? await db.search .findAll({ where: {id: ids, dimension}, @@ -572,6 +605,17 @@ module.exports = function(app) { const {dimension, drilldowns, id, limit = 5} = req.query; let {hierarchy} = req.query; + let cubes = { + "Geography": "usa_spending", + "University": "ipeds_completions", + "CIP": "ipeds_completions", + "NAPCS": "usa_spending", + "PUMS Industry": "pums_5", + "Industry": "pums_5", + "Occupation": "pums_5", + "PUMS Occupation": "pums_5" + } + if (dimension === "Geography") { const url = `${CANON_GEOSERVICE_API}neighbors/${id}`; @@ -607,8 +651,56 @@ module.exports = function(app) { res.json({data: attrs}); } - else { + else if (dimension === "Occupation" || dimension === "PUMS Occupation") { + let {include, filters} = req.query; + + const measure = req.query.measure || req.query.measures; + if (!measure.includes("Average Wage")) { + measure += ",Average Wage"; + } + delete req.query.dimension; + delete req.query.id; + delete req.query.hierarchy; + + const newQuery = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=pums_5&drilldowns=Year,${hierarchy}&measures=${measure}&time=Year.latest&sort=Average+Wage.desc${include ? `&include=${include}` : ""}${filters ? `&filters=${filters}` : ""}`; + const resp = await axios.get(newQuery) + .then(resp => { + if (resp.data && Array.isArray(resp.data)) { + resp.data.sort((a, b) => b["Average Wage"] - a["Average Wage"]); + } + return resp.data; + }) + .catch(error => ({error})); + + if (resp.error) res.json(resp); + else { + + const list = resp.data.sort((a, b) => b[measure.split(",")[0]] - a[measure.split(",")[0]]); + const entry = list.find(d => d[`${hierarchy} ID`] === id); + + const index = list.indexOf(entry); + let data; + + if (index <= limit / 2 + 1) { + data = list.slice(0, limit); + } + else if (index > list.length - limit / 2 - 1) { + data = list.slice(-limit); + } + else { + const min = Math.ceil(index - limit / 2); + data = list.slice(min, min + limit); + } + + data.forEach(d => { + d.Rank = list.indexOf(d) + 1; + }); + + res.json({data, source: resp.source}); + } + } + else { const where = {dimension, id}; if (hierarchy) where.hierarchy = hierarchy; const attr = await db.search.findOne({where}).catch(() => false); @@ -618,7 +710,7 @@ module.exports = function(app) { if (!hierarchy) hierarchy = attr.hierarchy; - req.query.limit = 10000; + req.query.limit = 5; const measure = req.query.measure || req.query.measures; if (req.query.measure) { req.query.measures = req.query.measure; @@ -641,18 +733,16 @@ module.exports = function(app) { if (measure !== "Obligation Amount" && !req.query.Year && !req.query.year) { const allYearQuery = Object.assign({[hierarchy]: id}, req.query); - const allYearParams = Object.entries(allYearQuery).map(([key, val]) => `${key}=${val}`).join("&"); - const allYearURL = `${CANON_API}/api/data?${allYearParams}`; + const allYearURL = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=${cubes[dimension]}&drilldowns=Year,${hierarchy}&include=${hierarchy}:${id}&measures=${allYearQuery.measures}&sort=${allYearQuery.order}.${allYearQuery.sort}`; const allYearData = await axios.get(allYearURL) .then(resp => resp.data) .catch(error => ({error})); - if (allYearData.error) req.query.Year = "latest"; - else req.query.Year = max(allYearData.data, d => d["ID Year"]); + if (allYearData.error) req.query.time = "&time=Year.latest"; + else req.query.Year = max(allYearData.data, d => d["Year"]); } const query = Object.assign({}, req.query); - const params = Object.entries(query).map(([key, val]) => `${key}=${val}`).join("&"); - const logicUrl = `${CANON_API}/api/data?${params}`; + const logicUrl = `${CANON_CONST_TESSERACT}tesseract/data.jsonrecords?cube=${cubes[dimension]}&drilldowns=Year,${hierarchy}&measures=${query.measures}&sort=${query.order}.${query.sort}${query.time ? query.time : ""}${query.Year ? `&include=Year:${query.Year}` : ""}`; const resp = await axios.get(logicUrl) .then(resp => resp.data) @@ -662,9 +752,9 @@ module.exports = function(app) { else { const list = resp.data; - const entry = list.find(d => d[`ID ${hierarchy}`] === id); - + const entry = list.find(d => d[`${hierarchy} ID`] == id); const index = list.indexOf(entry); + let data; if (index <= limit / 2 + 1) { @@ -683,12 +773,8 @@ module.exports = function(app) { }); res.json({data, source: resp.source}); - } } } - }); - - }; diff --git a/api/customAttributes.js b/api/customAttributes.js index 08a4ac2d7..72a37a445 100644 --- a/api/customAttributes.js +++ b/api/customAttributes.js @@ -83,9 +83,40 @@ module.exports = function(app) { retObj.stateDataID = state && !["Nation", "State"].includes(hierarchy) ? state.id : id; retObj.hierarchyElectionSub = ["Nation", "County"].includes(hierarchy) ? hierarchy : "State"; retObj.stateElectionId = ["Nation", "State", "County"].includes(hierarchy) ? id : stateElection.join(","); + retObj.hierarchyElectionSubTemp = ["Nation"].includes(hierarchy) ? hierarchy : "State"; + retObj.stateElectionIdTemp = ["Nation", "State"].includes(hierarchy) ? id : stateElection.join(","); retObj.electionCut = hierarchy === "Nation" ? `State` : hierarchy === "County" ? `County&State+County=${retObj.stateDataID}` : `County&State+County=${retObj.stateElectionId}`; retObj.hierarchySub = hierarchy === "Nation" ? "State" : "County"; retObj.CBPSection = hierarchy === "County" || (hierarchy === "State" && id !== "04000US72") || hierarchy === "MSA" + retObj.includeIncome = hierarchy === "Nation" ? `&exclude=State:0` : hierarchy === "State" ? `&include=State+County:${id}` : hierarchy === "County" ? `&include=County+Tract:${id}` : hierarchy === "Place" ? `&include=Place+Place-Tract:${id}` : ""; + retObj.incomeDrilldown = hierarchy === "Nation" ? "State" : hierarchy === "State" ? "County" : hierarchy === "County" ? "Tract" : hierarchy === "Place" ? "Place-Tract" : ""; + retObj.specialTessCut = hierarchy === "Nation" ? "&exclude=State:0" : hierarchy === "State" ? `&include=State+County:${id}` : hierarchy === "County" ? `&include=County+Tract:${id}` : hierarchy === "Place" ? `&include=State+Place:${state ? state.id : id}` : hierarchy === "MSA" ? `&include=State+County:${state ? state.id : id}` : hierarchy === "PUMA" ? `&include=State+PUMA:${state ? state.id : id}` : ""; + retObj.specialTessDrilldown = hierarchy === "Nation" ? "State" : hierarchy === "State" ? "County" : hierarchy === "MSA" ? "County" : hierarchy === "PUMA" ? "PUMA" : hierarchy === "County" ? "Tract" : hierarchy === "Place" ? "Place" : ""; + + if (hierarchy !== "Nation" && hierarchy !== "State" && hierarchy !== "PUMA") { + const url = `${CANON_GEOSERVICE_API}relations/intersects/${id}?targetLevels=state`; + const intersects = await axios.get(url) + .then(resp => resp.data) + .then(resp => { + if (resp.error) { + console.error(`[geoservice error] ${url}`); + console.error(resp.error); + return []; + } + else { + return resp || []; + } + }) + .then(resp => resp.map(d => d.geoid)) + .catch(() => []); + retObj.pumsID = intersects.join(","); + retObj.pumsHierarchy = "State"; + } + + else if (hierarchy === "PUMA" || hierarchy === "State" || hierarchy === "Nation") { + retObj.pumsID = id; + retObj.pumsHierarchy = hierarchy; + } if (hierarchy !== "Nation") { const url = `${CANON_GEOSERVICE_API}neighbors/${state ? state.id : id}`; @@ -105,6 +136,7 @@ module.exports = function(app) { .catch(() => []); retObj.stateNeighbors = neighbors.join(","); } + else { retObj.stateNeighbors = ""; } @@ -126,15 +158,10 @@ module.exports = function(app) { retObj.beaL0 = beaIds && beaIds.L0 ? beaIds.L0 : false; retObj.beaL1 = beaIds && beaIds.L1 ? beaIds.L1 : false; retObj.blsIds = blsInds[id] || false; - retObj.pumsLatestYear = await axios.get(`${origin}/api/data?${hierarchy}=${id}&measures=Total%20Population&limit=1&order=Year&sort=desc`) - .then(resp => resp.data.data[0]["ID Year"]) - .catch(() => 2020); + retObj.blsIdsStr = blsInds[id] ? blsInds[id].flat().join(",") : false; } else if (dimension === "PUMS Occupation") { retObj.blsIds = blsOccs[id] || false; - retObj.pumsLatestYear = await axios.get(`${origin}/api/data?${hierarchy}=${id}&measures=Total%20Population&limit=1&order=Year&sort=desc`) - .then(resp => resp.data.data[0]["ID Year"]) - .catch(() => 2020); } else if (dimension === "CIP") { retObj.stem = id.length === 6 ? stems.includes(id) ? "Stem Major" : false : "Contains Stem Majors"; diff --git a/api/homeRoute.js b/api/homeRoute.js index dcb2b267b..69c7780ab 100644 --- a/api/homeRoute.js +++ b/api/homeRoute.js @@ -257,4 +257,15 @@ module.exports = function(app) { }); + const {measures} = app.settings.cache; + const {homeGeomap} = app.settings.cache; + + app.get("/api/measures", async(req, res) => { + res.json(measures); + }); + + app.get("/api/home-geomap", async(req, res) => { + res.json(homeGeomap); + }); + }; diff --git a/app/App.jsx b/app/App.jsx index 518dc6d6d..81e353ba3 100644 --- a/app/App.jsx +++ b/app/App.jsx @@ -114,19 +114,7 @@ App.childContextTypes = { App.need = [ fetchData("formatters", "/api/formatters"), - fetchData("measures", "/api/cubes/", resp => { - const obj = {}; - for (const measure in resp.measures) { - if ({}.hasOwnProperty.call(resp.measures, measure)) { - const annotations = resp.measures[measure].annotations; - const format = annotations.error_for_measure - ? resp.measures[annotations.error_for_measure].annotations.units_of_measurement - : annotations.units_of_measurement; - obj[measure] = format; - } - } - return obj; - }) + fetchData("measures", "/api/measures") ]; export default connect(state => ({ diff --git a/app/components/Footer/index.css b/app/components/Footer/index.css index 035697fbf..2f7741a03 100644 --- a/app/components/Footer/index.css +++ b/app/components/Footer/index.css @@ -33,7 +33,7 @@ width: 373px; display: flex; flex-direction: row; - justify-content: space-between; + justify-content: space-evenly; margin: 0 auto; @media (max-width: 768px) { display: block; diff --git a/app/components/Footer/index.jsx b/app/components/Footer/index.jsx index 19f3f0771..726c10f04 100644 --- a/app/components/Footer/index.jsx +++ b/app/components/Footer/index.jsx @@ -14,8 +14,8 @@ export default class Footer extends Component {
Home
Reports
-
VizBuilder
-
Maps
+ {/*
VizBuilder
+
Maps
*/}
About
diff --git a/app/components/Nav/index.css b/app/components/Nav/index.css index a3f4a5e5a..aa2c97790 100644 --- a/app/components/Nav/index.css +++ b/app/components/Nav/index.css @@ -45,7 +45,7 @@ box-sizing: content-box; flex: none; height: 19px; - padding: 3px 10px 7px; + padding: 3px 7px; & img { height: 19px; width: 76px; diff --git a/app/components/Nav/index.jsx b/app/components/Nav/index.jsx index 79d34c936..3231006be 100644 --- a/app/components/Nav/index.jsx +++ b/app/components/Nav/index.jsx @@ -62,26 +62,26 @@ class Nav extends Component { const dark = !splash; - const Cart = () =>
-
Data Cart
- { cart && cart.data.length - ?
-
- { cart.data.length } Dataset{ cart.data.length > 1 ? "s" : "" } -
- { cart.data.map(d =>
-
{d.title}
- -
) } - - View Data - -
- Clear Cart -
-
- :
Put data into your cart as you browse to merge data from multiple sources.
} -
; + // const Cart = () =>
+ //
Data Cart
+ // { cart && cart.data.length + // ?
+ //
+ // { cart.data.length } Dataset{ cart.data.length > 1 ? "s" : "" } + //
+ // { cart.data.map(d =>
+ //
{d.title}
+ // + //
) } + // + // View Data + // + //
+ // Clear Cart + //
+ //
+ // :
Put data into your cart as you browse to merge data from multiple sources.
} + //
; return ; } - } export default connect(state => ({ diff --git a/app/components/Viz/Options.jsx b/app/components/Viz/Options.jsx index 9324168e6..c7289d955 100644 --- a/app/components/Viz/Options.jsx +++ b/app/components/Viz/Options.jsx @@ -362,7 +362,7 @@ class Options extends Component { const cartSize = cart ? cart.data.length : 0; const inCart = cart ? cart.data.find(c => c.slug === cartSlug) : false; - const cartEnabled = data && slug && title; + const cartEnabled = false; const shareEnabled = topic.slug; const baseURL = (typeof window === "undefined" ? location : window.location).href.split("#")[0].split("/").slice(0, 6).join("/"); const profileURL = `${baseURL}#${topic.slug}`; @@ -406,14 +406,14 @@ class Options extends Component { ; - const columns = results ? Object.keys(results[0]).filter(d => d.indexOf("ID ") === -1 && d.indexOf("Slug ") === -1) : []; + const columns = results ? Object.keys(results[0]).filter(d => d.indexOf(" ID") === -1 && d.indexOf("Slug ") === -1) : []; const stickies = ["Year", "Geography", "PUMS Industry", "PUMS Occupation", "CIP", "University", "Gender"].reverse(); columns.sort((a, b) => stickies.indexOf(b) - stickies.indexOf(a)); const columnWidths = columns.map(key => { if (key === "Year") return 60; else if (key.includes("Year")) return 150; - else if (key.includes("ID ")) return 120; + else if (key.includes(" ID")) return 120; else if (key.includes("University") || key.includes("Insurance")) return 250; else if (key.includes("Gender") || key.includes("Sex")) return 100; else if (stickies.includes(key)) return 200; @@ -511,11 +511,13 @@ Options.contextTypes = { formatters: PropTypes.object }; -export default connect(state => ({ - cart: state.cart, - location: state.location, - measures: state.data.measures -}), dispatch => ({ +export default connect(state => { + return { + cart: state.cart, + location: state.location, + measures: state.data.measures + }; +}, dispatch => ({ addToCart: build => dispatch(addToCart(build)), removeFromCart: build => dispatch(removeFromCart(build)) }))(Options); diff --git a/app/pages/Data/API.jsx b/app/pages/Data/API.jsx index f0f623edc..caf5e164f 100644 --- a/app/pages/Data/API.jsx +++ b/app/pages/Data/API.jsx @@ -9,7 +9,8 @@ export default class API extends Component {

Introduction

- The Data USA API allows users to explore the entire database using carefully constructed query strings, returning data as JSON results. All of the visualizations on the page have a "show data" button on their top-right that displays the API call(s) used to generate that visualization. Additionally, the new Viz Builder is a great way to explore what's possible. This page illustrates an example usage of exploring geographic data. + The Data USA API allows users to explore the entire database using carefully constructed query strings, returning data as JSON results. All of the visualizations on the page have a "show data" button on their top-right that displays the API call(s) used to generate that visualization. + {/* Additionally, the new Viz Builder is a great way to explore what's possible. This page illustrates an example usage of exploring geographic data. */}

Example: Population Data

diff --git a/app/pages/Home/index.jsx b/app/pages/Home/index.jsx index ba8f16574..f4fd9be47 100644 --- a/app/pages/Home/index.jsx +++ b/app/pages/Home/index.jsx @@ -7,7 +7,6 @@ import {Geomap} from "d3plus-react"; import SVG from "react-inlinesvg"; import "./index.css"; - import {format} from "d3-format"; const commas = format(","); @@ -84,15 +83,15 @@ class Home extends Component { /> d.State, legend: false, loadingHTML: "", ocean: "transparent", on: { click: d => { - router.push(`/profile/geo/${d["Slug State"]}`); + router.push(`/profile/geo/${d["slug"]}`); } }, projection: @@ -164,7 +163,7 @@ class Home extends Component {

- Merge and download data + Download data
@@ -203,7 +202,7 @@ class Home extends Component { -
+ {/*

The most powerful tools
@@ -255,7 +254,7 @@ class Home extends Component { }} />

- + */} ); @@ -265,7 +264,11 @@ class Home extends Component { } Home.need = [ - fetchData("home", "/api/home") + fetchData("home", "/api/home"), ]; -export default connect(state => ({tiles: state.data.home}))(Home); +export default connect( + state => ({ + tiles: state.data.home, + }) +)(Home); diff --git a/app/routes.jsx b/app/routes.jsx index de42f384a..91a568678 100644 --- a/app/routes.jsx +++ b/app/routes.jsx @@ -71,11 +71,11 @@ export default function RouteCreate() { - + {/* - + */} - + {/* */} diff --git a/cache/datasets.js b/cache/datasets.js index 441c12cbc..aaa71eda5 100644 --- a/cache/datasets.js +++ b/cache/datasets.js @@ -1,33 +1,38 @@ const axios = require("axios"); const {merge} = require("d3-array"); const {nest} = require("d3-collection"); -const {CANON_LOGICLAYER_CUBE} = process.env; -const prefix = `${CANON_LOGICLAYER_CUBE}${CANON_LOGICLAYER_CUBE.slice(-1) === "/" ? "" : "/"}`; +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; module.exports = async function() { - return axios.get(`${prefix}cubes`) + return axios.get(`${prefix}tesseract/cubes`) .then(resp => resp.data) - .then(data => nest() - .key(d => d.source_name) - .entries(data.cubes.map(d => d.annotations)) - .map(group => ({ - title: group.key, - desc: Array.from(new Set(group.values.map(d => d.source_description))), - datasets: nest() - .key(d => d.dataset_name) - .entries(group.values.filter(d => d.dataset_name)) - .map(g => ({ - title: g.key, - link: g.values[0].dataset_link, - tables: Array.from(new Set(merge(g.values.map(d => (d.table_id || "").split(",").filter(n => n.length))))) - })) - })) - .sort((a, b) => a.title.localeCompare(b.title))) + .then(data => { + const annotations = data.cubes + .map(d => d.annotations) + .filter(Boolean) + + return nest() + .key(d => d.source_name) + .entries(annotations) + .map(group => ({ + title: group.key, + desc: Array.from(new Set(group.values.map(d => d.source_description))), + datasets: nest() + .key(d => d.dataset_name) + .entries(group.values.filter(d => d.dataset_name)) + .map(g => ({ + title: g.key, + link: g.values[0].dataset_link, + tables: Array.from(new Set(merge(g.values.map(d => (d.table_id || "").split(",").filter(n => n.length))))) + })) + })) + .sort((a, b) => a.title.localeCompare(b.title)); + }) .catch(err => { console.error(` 🌎 Dataset Cache Error: ${err.message}`); if (err.config) console.error(err.config.url); return []; }); - }; diff --git a/cache/homeGeomap.js b/cache/homeGeomap.js new file mode 100644 index 000000000..684d262ee --- /dev/null +++ b/cache/homeGeomap.js @@ -0,0 +1,44 @@ +const axios = require("axios"); + +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; + +module.exports = async function (app) { + + const {db} = app; + + const geomapData = await axios.get(`${prefix}tesseract/data.jsonrecords?cube=acs_yg_total_population_5&drilldowns=State,Year&locale=en&measures=Population&time=Year.latest`) + .then(resp => resp.data) + .then(data => ({ data: data.data })) + .catch(err => { + console.error(` 🌎 Geomap Cache Error: ${err.message}`); + if (err.config) console.error(err.config.url); + return { data: [] }; + }); + + const stateRows = await db.search + .findAll({ + attributes: ["id", "slug"], + where: { + dimension: "Geography", + hierarchy: "State" + } + }).catch(err => { + console.error("Error fetching state slugs:", err.message); + return []; + }); + + const idToSlug = {}; + stateRows.forEach(row => { + idToSlug[row.id] = row.slug; + }); + + const dataWithSlugs = (geomapData.data || []).map(entry => ({ + ...entry, + slug: idToSlug[entry["State ID"]] + })); + + return { + data: dataWithSlugs, + }; +}; diff --git a/cache/measures.js b/cache/measures.js new file mode 100644 index 000000000..5a4e30bab --- /dev/null +++ b/cache/measures.js @@ -0,0 +1,36 @@ +const axios = require("axios"); +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; + +module.exports = async function () { + + return axios.get(`${prefix}tesseract/cubes`) + .then(resp => resp.data) + .then(resp => { + const obj = {}; + const measureMap = resp.cubes.reduce((acc, cube) => { + const cubeMeasureMap = cube.measures.reduce((mAcc, measure) => { + mAcc[measure.name] = measure.annotations; + return mAcc; + }, {}); + + return {...acc, ...cubeMeasureMap}; + }, {}) + + Object.keys(measureMap).forEach(m => { + const measure = measureMap[m]; + if (measure && measure.error_for_measure) { + const ref = measureMap[measure.error_for_measure]; + obj[m] = ref && ref.units_of_measurement ? ref.units_of_measurement : undefined; + } else { + obj[m] = measure && measure.units_of_measurement ? measure.units_of_measurement : undefined; + } + }); + + return obj; + }) + .catch(err => { + console.error(` 🌎 Measures Cache Error: ${err.message}`); + if (err.config) console.error(err.config.url); + }); +}; diff --git a/cache/opeid.js b/cache/opeid.js index c4f9acde4..2a74643fb 100644 --- a/cache/opeid.js +++ b/cache/opeid.js @@ -1,18 +1,20 @@ const axios = require("axios"); -const {CANON_LOGICLAYER_CUBE} = process.env; -const prefix = `${CANON_LOGICLAYER_CUBE}${CANON_LOGICLAYER_CUBE.slice(-1) === "/" ? "" : "/"}`; +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; -module.exports = async function() { +module.exports = async function () { - return axios.get(`${prefix}cubes/ipeds_completions/dimensions/University/hierarchies/University/levels/University/members?member_properties[]=OPEID6`) + return axios.get(`${prefix}tesseract/data.jsonrecords?cube=university_cube&drilldowns=University,OPEID6&locale=en&measures=Count`) .then(resp => resp.data) - .then(data => data.members.reduce((acc, d) => { - acc[d.key] = d.properties.OPEID6; - return acc; - }, {})) + .then(data => { + const result = data.data.reduce((acc, d) => { + acc[d["University ID"]] = d.OPEID6; + return acc; + }, {}); + return result; + }) .catch(err => { console.error(` 🌎 OPEID6 Cache Error: ${err.message}`); if (err.config) console.error(err.config.url); }); - }; diff --git a/cache/parents.js b/cache/parents.js index 91a871df9..3d3a12bfe 100644 --- a/cache/parents.js +++ b/cache/parents.js @@ -1,46 +1,108 @@ const axios = require("axios"); +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; -const {CANON_LOGICLAYER_CUBE} = process.env; -const prefix = `${CANON_LOGICLAYER_CUBE}${CANON_LOGICLAYER_CUBE.slice(-1) === "/" ? "" : "/"}`; - -/** Prases the return from the members call into an object lookup. */ -function parseParents(data) { - const obj = {}; - const levels = data.hierarchies[0].levels.filter(level => level.name !== "(All)"); - for (let i = 0; i < levels.length; i++) { - const members = levels[i].members; - for (let x = 0; x < members.length; x++) { - const d = members[x]; - const list = d.ancestors - .filter(a => a.level_name !== "(All)") - .map(a => `${a.key}`); - obj[d.key] = Array.from(new Set(list)); +/** + * Parses tesseract's members format into a flat lookup of key -> [parent keys] + * @param {Array} levelsData - Array of API responses for each dimension level + * @returns {Object} - { key: [parentKey1, parentKey2, ...] } + */ +function parseFlatParents(levelsData, isCIP = false) { + const lookup = {}; + for (const level of levelsData) { + for (const member of level.members) { + let key = String(member.key); + if (isCIP && (/^\d{1}$/.test(key) || /^\d{3}$/.test(key) || /^\d{5}$/.test(key))) { + key = `0${key}`; + } + const parentKeys = Array.from( + new Set( + (member.ancestor || []) + .map(a => { + let parentKey = String(a.key); + if (isCIP && (/^\d{1}$/.test(parentKey) || /^\d{3}$/.test(parentKey) || /^\d{5}$/.test(parentKey))) { + parentKey = `0${parentKey}`; + } + return parentKey; + }) + .filter(k => k !== key) + ) + ); + lookup[key] = parentKeys; } } - return obj; + return lookup; } -module.exports = function() { - - return Promise - .all([ - axios.get(`${prefix}cubes/pums_5/dimensions/PUMS%20Industry/`).then(resp => resp.data), - axios.get(`${prefix}cubes/pums_5/dimensions/PUMS%20Occupation/`).then(resp => resp.data), - axios.get(`${prefix}cubes/ipeds_completions/dimensions/CIP/`).then(resp => resp.data), - axios.get(`${prefix}cubes/ipeds_completions/dimensions/University/`).then(resp => resp.data), - axios.get(`${prefix}cubes/usa_spending/dimensions/NAPCS/`).then(resp => resp.data) - ]) - .then(([industries, occupations, courses, universities, products]) => ({ - naics: parseParents(industries), - soc: parseParents(occupations), - cip: parseParents(courses), - university: parseParents(universities), - napcs: parseParents(products) - })) - .catch(err => { - console.error(` 🌎 Parents Cache Error: ${err.message}`); - if (err.config) console.error(err.config.url); - return []; - }); +module.exports = async function () { + try { + // INDUSTRY + const industryEndpoints = [ + "tesseract/members?cube=pums_5&level=Industry%20Sector&parents=true", + "tesseract/members?cube=pums_5&level=Industry%20Sub-Sector&parents=true", + "tesseract/members?cube=pums_5&level=Industry%20Group&parents=true" + ]; + // OCCUPATION + const occupationEndpoints = [ + "tesseract/members?cube=pums_5&level=Major%20Occupation%20Group&parents=true", + "tesseract/members?cube=pums_5&level=Minor%20Occupation%20Group&parents=true", + "tesseract/members?cube=pums_5&level=Broad%20Occupation&parents=true", + "tesseract/members?cube=pums_5&level=Detailed%20Occupation&parents=true" + ]; + + // UNIVERSITY + const universityEndpoints = [ + "tesseract/members?cube=ipeds_completions&level=Carnegie+Parent&parents=true", + "tesseract/members?cube=ipeds_completions&level=Carnegie&parents=true", + "tesseract/members?cube=ipeds_completions&level=University&parents=true" + ]; + + // CIP + const cipEndpoints = [ + "tesseract/members?cube=ipeds_completions&level=CIP2&parents=true", + "tesseract/members?cube=ipeds_completions&level=CIP4&parents=true", + "tesseract/members?cube=ipeds_completions&level=CIP6&parents=true" + ]; + + // NAPCS + const napcsEndpoints = [ + "tesseract/members?cube=usa_spending&level=NAPCS+Section&parents=true", + "tesseract/members?cube=usa_spending&level=NAPCS+Group&parents=true", + "tesseract/members?cube=usa_spending&level=NAPCS+Class&parents=true" + ]; + + // Fetch all endpoints in parallel + const [ + ...industries + ] = await Promise.all(industryEndpoints.map(url => axios.get(prefix + url).then(r => r.data))); + + const [ + ...occupations + ] = await Promise.all(occupationEndpoints.map(url => axios.get(prefix + url).then(r => r.data))); + + const [ + ...universities + ] = await Promise.all(universityEndpoints.map(url => axios.get(prefix + url).then(r => r.data))); + + const [ + ...courses + ] = await Promise.all(cipEndpoints.map(url => axios.get(prefix + url).then(r => r.data))); + + const [ + ...products + ] = await Promise.all(napcsEndpoints.map(url => axios.get(prefix + url).then(r => r.data))); + + return { + naics: parseFlatParents(industries), + soc: parseFlatParents(occupations), + cip: parseFlatParents(courses, true), + university: parseFlatParents(universities), + napcs: parseFlatParents(products) + }; + } catch (err) { + console.error(` 🌎 Parents Cache Error: ${err.message}`); + if (err.config) console.error(err.config.url); + return []; + } }; diff --git a/cache/pops.js b/cache/pops.js index e222eb702..8fd9fd9ef 100644 --- a/cache/pops.js +++ b/cache/pops.js @@ -1,30 +1,22 @@ -const {Client, MondrianDataSource} = require("@datawheel/olap-client"); -const {CANON_LOGICLAYER_CUBE} = process.env; +const axios = require("axios"); +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; module.exports = async function() { - const datasource = new MondrianDataSource(CANON_LOGICLAYER_CUBE); - const client = new Client(datasource); - const levels = ["Nation", "State", "County", "MSA", "Place", "PUMA", "Congressional District"]; - const popQueries = levels - .map(level => client.getCube("acs_yg_total_population_5") - .then(c => { - const query = c.query - .addDrilldown(level) - .addMeasure("Population") - .addCut("[Year].[Year]", ["2019"]) - .setFormat("jsonrecords"); - return client.execQuery(query); - }) - .then(resp => resp.data.reduce((acc, d) => { - acc[d[`ID ${level}`]] = d.Population; + const popQueries = levels.map(level => { + const url = `${prefix}tesseract/data.jsonrecords?cube=acs_yg_total_population_5&drilldowns=${encodeURIComponent(level)}&locale=en&measures=Population&time=Year.latest`; + return axios.get(url) + .then(resp => resp.data.data.reduce((acc, d) => { + acc[d[`${level} ID`]] = d.Population; return acc; }, {})) .catch(err => { console.error(` 🌎 ${level} Pop Cache Error: ${err.message}`); if (err.config) console.error(err.config.url); - })); + }); + }); const rawPops = await Promise.all(popQueries); const pops = rawPops.reduce((obj, d) => (obj = Object.assign(obj, d), obj), {}); diff --git a/cache/sctg.js b/cache/sctg.js index fd9943db4..f45cac8c1 100644 --- a/cache/sctg.js +++ b/cache/sctg.js @@ -1,18 +1,17 @@ const axios = require("axios"); -const {CANON_LOGICLAYER_CUBE} = process.env; -const prefix = `${CANON_LOGICLAYER_CUBE}${CANON_LOGICLAYER_CUBE.slice(-1) === "/" ? "" : "/"}`; +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; -module.exports = function() { +module.exports = function () { - return axios.get(`${prefix}cubes/dot_faf/dimensions/SCTG/`) + return axios.get(`${prefix}tesseract/members?cube=dot_faf&level=SCTG2`) .then(resp => resp.data) .then(data => { - const {members} = data.hierarchies[0].levels[1]; - return members.reduce((obj, d) => { + return data.members.reduce((obj, d) => { obj[d.key] = { id: d.key, - name: d.caption || d.name + name: d.caption }; return obj; }, {}); @@ -22,5 +21,4 @@ module.exports = function() { if (err.config) console.error(err.config.url); return []; }); - }; diff --git a/cache/urls.js b/cache/urls.js index b3bee1cb8..1cbd508de 100644 --- a/cache/urls.js +++ b/cache/urls.js @@ -1,15 +1,18 @@ const axios = require("axios"); -const {CANON_LOGICLAYER_CUBE} = process.env; -const prefix = `${CANON_LOGICLAYER_CUBE}${CANON_LOGICLAYER_CUBE.slice(-1) === "/" ? "" : "/"}`; +const {CANON_CONST_TESSERACT} = process.env; +const prefix = `${CANON_CONST_TESSERACT}${CANON_CONST_TESSERACT.slice(-1) === "/" ? "" : "/"}`; -module.exports = async function() { +module.exports = async function () { - return axios.get(`${prefix}cubes/ipeds_completions/dimensions/University/hierarchies/University/levels/University/members?member_properties[]=URL`) + return axios.get(`${prefix}tesseract/data.jsonrecords?cube=university_cube&drilldowns=University,URL&locale=en&measures=Count`) .then(resp => resp.data) - .then(data => data.members.reduce((acc, d) => { - acc[d.key] = d.properties.URL; - return acc; - }, {})) + .then(data => { + const result = data.data.reduce((acc, d) => { + acc[d["University ID"]] = d.URL; + return acc; + }, {}); + return result; + }) .catch(err => { console.error(` 🌎 URL Cache Error: ${err.message}`); if (err.config) console.error(err.config.url); diff --git a/canon.js b/canon.js index 8fa8dbd71..2d0c36774 100644 --- a/canon.js +++ b/canon.js @@ -54,159 +54,159 @@ module.exports = { ] } ], - logiclayer: { - aliases: { - "CIP": "cip", - "Geography": "geo", - "measures": ["measure", "required"], - "PUMS Industry": "naics", - "PUMS Occupation": "soc", - "University": "university", - "Year": "year", - "NAPCS": "napcs" - }, - cubeFilters: [ - { - filter: cubes => { + // logiclayer: { + // aliases: { + // "CIP": "cip", + // "Geography": "geo", + // "measures": ["measure", "required"], + // "PUMS Industry": "naics", + // "PUMS Occupation": "soc", + // "University": "university", + // "Year": "year", + // "NAPCS": "napcs" + // }, + // cubeFilters: [ + // { + // filter: cubes => { - if (cubes.find(cube => cube.name.includes("_c_"))) { - cubes = cubes.filter(cube => cube.name.includes("_c_")); - } - else if (cubes.find(cube => cube.name.includes("_2016_"))) { - cubes = cubes.filter(cube => cube.name.includes("_2016_")); - } + // if (cubes.find(cube => cube.name.includes("_c_"))) { + // cubes = cubes.filter(cube => cube.name.includes("_c_")); + // } + // else if (cubes.find(cube => cube.name.includes("_2016_"))) { + // cubes = cubes.filter(cube => cube.name.includes("_2016_")); + // } - return cubes.length === 1 ? cubes : cubes.filter(cube => cube.name.match(/_5$/g)); + // return cubes.length === 1 ? cubes : cubes.filter(cube => cube.name.match(/_5$/g)); - }, - key: cube => cube.name.replace("_c_", "_").replace("_2016_", "_").replace(/_[0-9]$/g, "") - }, - { - filter: cubes => cubes.filter(c => c.name === "ipeds_graduation_demographics_v3"), - key: cube => !cube ? "test" : cube.name === "ipeds_undergrad_grad_rate_demographics" || cube.name === "ipeds_graduation_demographics_v2" ? "ipeds_graduation_demographics_v3" : cube.name - } - ], - dimensionMap: { - "CIP2": "CIP", - "Industry": "PUMS Industry", - "Commodity L0": "PUMS Industry", - // "Commodity L1": "PUMS Industry", - "Industry L0": "PUMS Industry", - // "Industry L1": "PUMS Industry", - "Occupation": "PUMS Occupation", - "OPEID": "University", - "SCTG2": "NAPCS", - "Destination State": "Geography", - "Origin State": "Geography" - }, - relations: { - "Destination State": geoRelations, - "Origin State": geoRelations, - "Geography": geoRelations, - "CIP": { - parents: { - url: id => `${CANON_API}/api/parents/cip/${id}`, - callback: arr => arr.map(d => d.id) - } - }, - "PUMS Industry": { - parents: { - url: id => `${CANON_API}/api/parents/naics/${id}`, - callback: arr => arr.map(d => d.id) - } - }, - "NAPCS": { - parents: { - url: id => `${CANON_API}/api/parents/napcs/${id}`, - callback: arr => arr.map(d => d.id) - } - }, - "PUMS Occupation": { - parents: { - url: id => `${CANON_API}/api/parents/soc/${id}`, - callback: arr => arr.map(d => d.id) - } - }, - "University": { - parents: { - url: id => `${CANON_API}/api/parents/university/${id}`, - callback: arr => arr.map(d => d.id) - }, - similar: { - url: id => `${CANON_API}/api/university/similar/${id}`, - callback: arr => arr.map(d => d.id) - } - } - }, - substitutions: { - "Geography": { - levels: { - "State": ["Nation"], - "County": ["MSA", "State", "Origin State", "Destination State", "Nation"], - "MSA": ["State", "Origin State", "Destination State", "Nation"], - "Place": ["County", "MSA", "State", "Origin State", "Destination State", "Nation"], - "PUMA": ["State", "Origin State", "Destination State", "Nation"] - }, - url: (id, level) => { - const targetLevel = level.toLowerCase(); - return `${CANON_GEOSERVICE_API}relations/intersects/${id}?targetLevels=${targetLevel}&overlapSize=true`; - }, - callback: resp => { - let arr = []; - if (resp.error) { - console.error("[geoservice error]"); - console.error(resp.error); - } - else { - arr = resp || []; - } - arr = arr.filter(d => d.overlap_size > 0.00001).sort((a, b) => b.overlap_size - a.overlap_size); - return arr.length ? arr.every(d => d.level === "state") ? arr.filter(d => d.level === "state").map(d => d.geoid) : arr[0].geoid : "01000US"; - } - }, - "CIP": { - levels: { - CIP6: ["CIP4", "CIP2"], - CIP4: ["CIP2"] - }, - url: (id, level) => `${CANON_API}/api/cip/parent/${id}/${level}/`, - callback: resp => resp.id - }, - "PUMS Industry": { - levels: { - "Industry Group": ["Industry", "Industry L0", "Commodity L0"], - "Industry Sub-Sector": ["Industry", "Industry L0", "Commodity L0"], - "Industry Sector": ["Industry", "Industry L0", "Commodity L0"] - }, - url: (id, level) => `${CANON_API}/api/naics/${id}/${level}`, - callback: resp => resp - }, - "PUMS Occupation": { - levels: { - "Major Occupation Group": ["Occupation"], - "Minor Occupation Group": ["Occupation"], - "Broad Occupation": ["Occupation"], - "Detailed Occupation": ["Occupation"] - }, - url: id => `${CANON_API}/api/soc/${id}/bls`, - callback: resp => resp - }, - "NAPCS": { - levels: { - "NAPCS Section": ["SCTG2"], - "NAPCS Group": ["SCTG2"], - "NAPCS Class": ["SCTG2"] - }, - url: id => `${CANON_API}/api/napcs/${id}/sctg/`, - callback: resp => resp.map(d => d.id) - }, - "University": { - levels: { - University: ["OPEID"] - }, - url: id => `${CANON_API}/api/university/opeid/${id}/`, - callback: resp => resp.opeid - } - } - } + // }, + // key: cube => cube.name.replace("_c_", "_").replace("_2016_", "_").replace(/_[0-9]$/g, "") + // }, + // { + // filter: cubes => cubes.filter(c => c.name === "ipeds_graduation_demographics_v3"), + // key: cube => !cube ? "test" : cube.name === "ipeds_undergrad_grad_rate_demographics" || cube.name === "ipeds_graduation_demographics_v2" ? "ipeds_graduation_demographics_v3" : cube.name + // } + // ], + // dimensionMap: { + // "CIP2": "CIP", + // "Industry": "PUMS Industry", + // "Commodity L0": "PUMS Industry", + // // "Commodity L1": "PUMS Industry", + // "Industry L0": "PUMS Industry", + // // "Industry L1": "PUMS Industry", + // "Occupation": "PUMS Occupation", + // "OPEID": "University", + // "SCTG2": "NAPCS", + // "Destination State": "Geography", + // "Origin State": "Geography" + // }, + // relations: { + // "Destination State": geoRelations, + // "Origin State": geoRelations, + // "Geography": geoRelations, + // "CIP": { + // parents: { + // url: id => `${CANON_API}/api/parents/cip/${id}`, + // callback: arr => arr.map(d => d.id) + // } + // }, + // "PUMS Industry": { + // parents: { + // url: id => `${CANON_API}/api/parents/naics/${id}`, + // callback: arr => arr.map(d => d.id) + // } + // }, + // "NAPCS": { + // parents: { + // url: id => `${CANON_API}/api/parents/napcs/${id}`, + // callback: arr => arr.map(d => d.id) + // } + // }, + // "PUMS Occupation": { + // parents: { + // url: id => `${CANON_API}/api/parents/soc/${id}`, + // callback: arr => arr.map(d => d.id) + // } + // }, + // "University": { + // parents: { + // url: id => `${CANON_API}/api/parents/university/${id}`, + // callback: arr => arr.map(d => d.id) + // }, + // similar: { + // url: id => `${CANON_API}/api/university/similar/${id}`, + // callback: arr => arr.map(d => d.id) + // } + // } + // }, + // substitutions: { + // "Geography": { + // levels: { + // "State": ["Nation"], + // "County": ["MSA", "State", "Origin State", "Destination State", "Nation"], + // "MSA": ["State", "Origin State", "Destination State", "Nation"], + // "Place": ["County", "MSA", "State", "Origin State", "Destination State", "Nation"], + // "PUMA": ["State", "Origin State", "Destination State", "Nation"] + // }, + // url: (id, level) => { + // const targetLevel = level.toLowerCase(); + // return `${CANON_GEOSERVICE_API}relations/intersects/${id}?targetLevels=${targetLevel}&overlapSize=true`; + // }, + // callback: resp => { + // let arr = []; + // if (resp.error) { + // console.error("[geoservice error]"); + // console.error(resp.error); + // } + // else { + // arr = resp || []; + // } + // arr = arr.filter(d => d.overlap_size > 0.00001).sort((a, b) => b.overlap_size - a.overlap_size); + // return arr.length ? arr.every(d => d.level === "state") ? arr.filter(d => d.level === "state").map(d => d.geoid) : arr[0].geoid : "01000US"; + // } + // }, + // "CIP": { + // levels: { + // CIP6: ["CIP4", "CIP2"], + // CIP4: ["CIP2"] + // }, + // url: (id, level) => `${CANON_API}/api/cip/parent/${id}/${level}/`, + // callback: resp => resp.id + // }, + // "PUMS Industry": { + // levels: { + // "Industry Group": ["Industry", "Industry L0", "Commodity L0"], + // "Industry Sub-Sector": ["Industry", "Industry L0", "Commodity L0"], + // "Industry Sector": ["Industry", "Industry L0", "Commodity L0"] + // }, + // url: (id, level) => `${CANON_API}/api/naics/${id}/${level}`, + // callback: resp => resp + // }, + // "PUMS Occupation": { + // levels: { + // "Major Occupation Group": ["Occupation"], + // "Minor Occupation Group": ["Occupation"], + // "Broad Occupation": ["Occupation"], + // "Detailed Occupation": ["Occupation"] + // }, + // url: id => `${CANON_API}/api/soc/${id}/bls`, + // callback: resp => resp + // }, + // "NAPCS": { + // levels: { + // "NAPCS Section": ["SCTG2"], + // "NAPCS Group": ["SCTG2"], + // "NAPCS Class": ["SCTG2"] + // }, + // url: id => `${CANON_API}/api/napcs/${id}/sctg/`, + // callback: resp => resp.map(d => d.id) + // }, + // "University": { + // levels: { + // University: ["OPEID"] + // }, + // url: id => `${CANON_API}/api/university/opeid/${id}/`, + // callback: resp => resp.opeid + // } + // } + // } }; diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 000000000..c6c366bda --- /dev/null +++ b/compose.yaml @@ -0,0 +1,38 @@ +services: + site: &app + container_name: site + image: ${GCP_ARTIFACT_REGISTRY_LOCATION}-docker.pkg.dev/${GCP_PROJECT_ID}/${GCP_ARTIFACT_REGISTRY_NAME}/${GCP_IMAGE_NAME}:${GCP_IMAGE_TAG} + restart: always + stop_signal: SIGTERM + stop_grace_period: 30s + volumes: + - "/home/${GCP_VM_USER}/${GCP_ARTIFACT_REGISTRY_NAME}-${GCP_IMAGE_NAME}/google:/app/google/" + env_file: + - .env.gcp + networks: + - site + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "3300:3300" + + site-1: &app + container_name: site-1 + image: ${GCP_ARTIFACT_REGISTRY_LOCATION}-docker.pkg.dev/${GCP_PROJECT_ID}/${GCP_ARTIFACT_REGISTRY_NAME}/${GCP_IMAGE_NAME}:${GCP_IMAGE_TAG} + restart: always + stop_signal: SIGTERM + stop_grace_period: 30s + volumes: + - "/home/${GCP_VM_USER}/${GCP_ARTIFACT_REGISTRY_NAME}-${GCP_IMAGE_NAME}/google:/app/google/" + env_file: + - .env.gcp + networks: + - site + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "3301:3300" + +networks: + site: + driver: bridge diff --git a/deploy_to_vm.sh b/deploy_to_vm.sh new file mode 100644 index 000000000..ff05ab3d9 --- /dev/null +++ b/deploy_to_vm.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Pull the latest version and create the containers if they don't exist +export $(cat .env.gcp | xargs) + +echo "Pulling the latest version and creating the containers if they don't exist" +docker compose --env-file .env.gcp pull + +echo "Compose down" +docker compose down + +echo "Compose up" +docker compose --env-file .env.gcp up -d + +# Clean old images with the 'project=site' label +docker image prune -af --filter="label=project=site" diff --git a/helm/development.yaml b/helm/development.yaml index 4ed238063..dd01839fe 100644 --- a/helm/development.yaml +++ b/helm/development.yaml @@ -74,7 +74,7 @@ serviceAccount: configMap: CANON_API: "https://app-dev.datausa.io" - CANON_CMS_CUBES: "https://gary-api.datausa.io/" + CANON_CMS_CUBES: "https://honolulu-api.datausa.io/" CANON_CMS_ENABLE: "true" CANON_CMS_FORCE_HTTPS: "true" CANON_CMS_GENERATOR_TIMEOUT: "600000" @@ -82,7 +82,7 @@ configMap: CANON_CMS_MINIMUM_ROLE: "1" CANON_CMS_REQUESTS_PER_SECOND: "60" CANON_CONST_CART: "datausa-cart-v3" - CANON_CONST_CUBE: "https://gary-api.datausa.io/" + CANON_CONST_CUBE: "https://honolulu-api.datausa.io/" CANON_CONST_TESSERACT: "https://api-ts-dev.datausa.io/" CANON_DB_NAME: "datausa-cms-21-dev" CANON_DB_USER: "postgres" @@ -90,7 +90,7 @@ configMap: CANON_GOOGLE_ANALYTICS: "UA-70325841-1" CANON_LANGUAGES: "en" CANON_LANGUAGE_DEFAULT: "en" - CANON_LOGICLAYER_CUBE: "https://gary-api.datausa.io/" + CANON_LOGICLAYER_CUBE: "https://honolulu-api.datausa.io/" CANON_LOGICLAYER_SLUGS: "true" CANON_LOGINS: "true" GA_KEYFILE: "/app/google/googleAnalyticsKey.json" @@ -120,12 +120,12 @@ ingress: paths: - / - /ws - # - host: gary-app.datausa.io - # paths: - # - / - # - /ws + - host: honolulu-app.datausa.io + paths: + - / + - /ws tls: - secretName: canon-site-tls hosts: - app-dev.datausa.io - # - gary-app.datausa.io + - honolulu-app.datausa.io diff --git a/helm/gary.yaml b/helm/gary.yaml index 1742c7b2e..e6ff034f7 100644 --- a/helm/gary.yaml +++ b/helm/gary.yaml @@ -135,6 +135,9 @@ ingress: if ($request_uri ~* "/api/profile/\?slug=(geo|soc|naics|napcs|university|cip)") { set $no_use_cache 0; } + if ($request_uri ~* "/api/(geo|soc|naics|napcs|university|cip)/similar") { + set $no_use_cache 0; + } if ($request_uri ~* "^/$") { set $no_use_cache 0; } diff --git a/jhcovid19.py b/jhcovid19.py deleted file mode 100644 index dee2ac8c0..000000000 --- a/jhcovid19.py +++ /dev/null @@ -1,58 +0,0 @@ -import pandas as pd -import os -import datetime as dt - -today = dt.date.today() -start_day = dt.date(2020, 1, 22) -diff = today - start_day -days = diff.days -dates = pd.date_range("2020-01-22", periods=days+1, - freq="D").strftime('%m-%d-%Y') -dates = pd.Series(dates).astype(str) - -data = [] -for date in dates: - - date_ = pd.to_datetime(date) - - try: - url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/{}.csv".format( - date) - df = pd.read_csv(url, sep=",") - - if date_ <= pd.to_datetime("2020-03-21"): - - df = df[["Province/State", "Country/Region", - "Confirmed", "Deaths", "Recovered"]] - df = df.rename(columns={"Country/Region": "Geography"}) - - else: - - df = df[["Province_State", "Country_Region", - "Confirmed", "Deaths", "Recovered"]] - df = df.rename(columns={"Country_Region": "Geography"}) - - df["Date"] = date - df["Date"] = df["Date"].str[6:10] + "/" + \ - df["Date"].str[0:2] + "/" + df["Date"].str[3:5] - df = df[df["Geography"] != "US"] - df["Geography"] = df["Geography"].replace("Mainland China", "China") - df[["Confirmed", "Deaths", "Recovered"]] = df[[ - "Confirmed", "Deaths", "Recovered"]].astype("Int64") - - data.append(df) - except Exception as ex: - print(date, ex) - - -data = pd.concat(data, sort=False) - -data = data.groupby(["Geography", "Date"]).sum().reset_index() -data["ID Geography"] = data["Geography"] - -path = os.path.dirname(os.path.abspath("__file__")) + \ - "/static/datacovid19.json" - -previous = pd.read_json(path) if os.path.exists(path) else pd.DataFrame([]) -if len(data) > len(previous): - data.to_json(path, orient="records") diff --git a/mobilitycovid19.py b/mobilitycovid19.py deleted file mode 100644 index 3cd666a29..000000000 --- a/mobilitycovid19.py +++ /dev/null @@ -1,69 +0,0 @@ -import pandas as pd -import os - -stateToFips = {"AL": "04000US01", "AK": "04000US02", "AZ": "04000US04", "AR": "04000US05", "CA": "04000US06", - "CO": "04000US08", "CT": "04000US09", "DE": "04000US10", "DC": "04000US11", "FL": "04000US12", - "GA": "04000US13", "HI": "04000US15", "ID": "04000US16", "IL": "04000US17", "IN": "04000US18", - "IA": "04000US19", "KS": "04000US20", "KY": "04000US21", "LA": "04000US22", "ME": "04000US23", - "MD": "04000US24", "MA": "04000US25", "MI": "04000US26", "MN": "04000US27", "MS": "04000US28", - "MO": "04000US29", "MT": "04000US30", "NE": "04000US31", "NV": "04000US32", "NH": "04000US33", - "NJ": "04000US34", "NM": "04000US35", "NY": "04000US36", "NC": "04000US37", "ND": "04000US38", - "OH": "04000US39", "OK": "04000US40", "OR": "04000US41", "PA": "04000US42", "RI": "04000US44", - "SC": "04000US45", "SD": "04000US46", "TN": "04000US47", "TX": "04000US48", "UT": "04000US49", - "VT": "04000US50", "VA": "04000US51", "WA": "04000US53", "WV": "04000US54", "WI": "04000US55", - "WY": "04000US56"} - -states = {"Alabama": "AL", "Alaska": "AK", "Arizona": "AZ", "Arkansas": "AR", "California": "CA", "Colorado": "CO", - "Connecticut": "CT", "District of Columbia": "DC", "Delaware": "DE", "Florida": "FL", "Georgia": "GA", - "Hawaii": "HI", "Idaho": "ID", "Illinois": "IL", "Indiana": "IN", "Iowa": "IA", "Kansas": "KS", - "Kentucky": "KY", "Louisiana": "LA", "Maine": "ME", "Maryland": "MD", "Massachusetts": "MA", "Michigan": "MI", - "Minnesota": "MN", "Mississippi": "MS", "Missouri": "MO", "Montana": "MT", "Nebraska": "NE", "Nevada": "NV", - "New Hampshire": "NH", "New Jersey": "NJ", "New Mexico": "NM", "New York": "NY", "North Carolina": "NC", - "North Dakota": "ND", "Ohio": "OH", "Oklahoma": "OK", "Oregon": "OR", "Pennsylvania": "PA", - "Rhode Island": "RI", "South Carolina": "SC", "South Dakota": "SD", "Tennessee": "TN", "Texas": "TX", - "Utah": "UT", "Vermont": "VT", "Virginia": "VA", "Washington": "WA", "West Virginia": "WV", - "Wisconsin": "WI", "Wyoming": "WY", "Chicago": "IL"} - -df_google = pd.read_csv("https://www.gstatic.com/covid19/mobility/Global_Mobility_Report.csv", low_memory=False) - -df_google = df_google[df_google["country_region_code"] == "US"] -df_google = df_google[(~df_google["sub_region_1"].isna()) & (df_google["sub_region_2"].isna())] - -df_google = df_google.melt( - id_vars=["country_region", "sub_region_1", "date"], - value_vars=[ - "retail_and_recreation_percent_change_from_baseline", - "grocery_and_pharmacy_percent_change_from_baseline", - "parks_percent_change_from_baseline", - "transit_stations_percent_change_from_baseline", - "workplaces_percent_change_from_baseline", - "residential_percent_change_from_baseline" - ] -) - -df_google["variable"] = df_google["variable"].replace({ - "retail_and_recreation_percent_change_from_baseline": "Retail and Recreation", - "grocery_and_pharmacy_percent_change_from_baseline": "Grocery and Pharmacy", - "parks_percent_change_from_baseline": "Parks", - "transit_stations_percent_change_from_baseline": "Transit Stations", - "workplaces_percent_change_from_baseline": "Workplaces", - "residential_percent_change_from_baseline": "Residential" -}) - -df_google = df_google.drop(columns=["country_region"]) -df_google = df_google.rename(columns={ - "sub_region_1": "Geography", - "date": "Date", - "variable": "Type", - "value": "Percent Change from Baseline" -}) - -df_google = df_google[~df_google["Geography"].isna()] -df_google["ID Geography"] = df_google["Geography"].replace(states).replace(stateToFips) -df_google["Date"] = df_google["Date"].str.replace("-", "/") - -path = os.path.dirname(os.path.abspath("__file__")) + "/static/mobilitycovid19.json" - -previous = pd.read_json(path) if os.path.exists(path) else pd.DataFrame([]) -if len(df_google) > len(previous): - df_google.to_json(path, orient="records") diff --git a/package-lock.json b/package-lock.json index f8ceecea9..0305a59c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@blueprintjs/table": "^3.8.33", "@datawheel/canon-cms": "^0.22.10", "@datawheel/canon-core": "^0.26.1", - "@datawheel/canon-logiclayer": "^0.6.1", "@datawheel/canon-vizbuilder": "^0.5.3", "@datawheel/olap-client": "^2.0.0-beta.3", "@google-analytics/data": "^4.4.0", @@ -2549,34 +2548,6 @@ "node": ">=0.10.0" } }, - "node_modules/@datawheel/canon-logiclayer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@datawheel/canon-logiclayer/-/canon-logiclayer-0.6.1.tgz", - "integrity": "sha512-766djwaj0+kF8WrFmRNPa1rhAVA88JWzzWtme1cQubZG5uJ6cX9/Lf4EXJvkLERXQKSWFbp+KX1khdyBXDOUbQ==", - "dependencies": { - "@blueprintjs/core": "^3.45.0", - "axios": "^0.21.1", - "d3-array": "^2.11.0", - "d3-collection": "^1.0.4", - "d3plus-common": "~1.1.2", - "mondrian-rest-client": "^1.1.4", - "perfect-express-sanitizer": "^1.0.9", - "promise-throttle": "^1.0.0", - "sequelize": "^5", - "yn": "^4.0.0" - }, - "peerDependencies": { - "@datawheel/canon-core": "^0.26.0" - } - }, - "node_modules/@datawheel/canon-logiclayer/node_modules/yn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz", - "integrity": "sha512-huWiiCS4TxKc4SfgmTwW1K7JmXPPAmuXWYy4j9qjQo4+27Kni8mGhAAi1cloRWmBe2EqcLgt3IGqQoRL/MtPgg==", - "engines": { - "node": ">=10" - } - }, "node_modules/@datawheel/canon-vizbuilder": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/@datawheel/canon-vizbuilder/-/canon-vizbuilder-0.5.3.tgz", @@ -27004,30 +26975,6 @@ } } }, - "@datawheel/canon-logiclayer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@datawheel/canon-logiclayer/-/canon-logiclayer-0.6.1.tgz", - "integrity": "sha512-766djwaj0+kF8WrFmRNPa1rhAVA88JWzzWtme1cQubZG5uJ6cX9/Lf4EXJvkLERXQKSWFbp+KX1khdyBXDOUbQ==", - "requires": { - "@blueprintjs/core": "^3.45.0", - "axios": "^0.21.1", - "d3-array": "^2.11.0", - "d3-collection": "^1.0.4", - "d3plus-common": "~1.1.2", - "mondrian-rest-client": "^1.1.4", - "perfect-express-sanitizer": "^1.0.9", - "promise-throttle": "^1.0.0", - "sequelize": "^5", - "yn": "^4.0.0" - }, - "dependencies": { - "yn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz", - "integrity": "sha512-huWiiCS4TxKc4SfgmTwW1K7JmXPPAmuXWYy4j9qjQo4+27Kni8mGhAAi1cloRWmBe2EqcLgt3IGqQoRL/MtPgg==" - } - } - }, "@datawheel/canon-vizbuilder": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/@datawheel/canon-vizbuilder/-/canon-vizbuilder-0.5.3.tgz", diff --git a/package.json b/package.json index ffaf3eb42..c59b3d9b1 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "@blueprintjs/table": "^3.8.33", "@datawheel/canon-cms": "^0.22.10", "@datawheel/canon-core": "^0.26.1", - "@datawheel/canon-logiclayer": "^0.6.1", "@datawheel/canon-vizbuilder": "^0.5.3", "@datawheel/olap-client": "^2.0.0-beta.3", "@google-analytics/data": "^4.4.0", diff --git a/usacovid19.py b/usacovid19.py deleted file mode 100644 index 0b0e590ad..000000000 --- a/usacovid19.py +++ /dev/null @@ -1,113 +0,0 @@ - -import pandas as pd -import datetime as dt -import numpy as np -import os - -stateToDivision = {"AL": "04000US01", "AK": "04000US02", "AZ": "04000US04", "AR": "04000US05", "CA": "04000US06", - "CO": "04000US08", "CT": "04000US09", "DE": "04000US10", "DC": "04000US11", "FL": "04000US12", - "GA": "04000US13", "HI": "04000US15", "ID": "04000US16", "IL": "04000US17", "IN": "04000US18", - "IA": "04000US19", "KS": "04000US20", "KY": "04000US21", "LA": "04000US22", "ME": "04000US23", - "MD": "04000US24", "MA": "04000US25", "MI": "04000US26", "MN": "04000US27", "MS": "04000US28", - "MO": "04000US29", "MT": "04000US30", "NE": "04000US31", "NV": "04000US32", "NH": "04000US33", - "NJ": "04000US34", "NM": "04000US35", "NY": "04000US36", "NC": "04000US37", "ND": "04000US38", - "OH": "04000US39", "OK": "04000US40", "OR": "04000US41", "PA": "04000US42", "RI": "04000US44", - "SC": "04000US45", "SD": "04000US46", "TN": "04000US47", "TX": "04000US48", "UT": "04000US49", - "VT": "04000US50", "VA": "04000US51", "WA": "04000US53", "WV": "04000US54", "WI": "04000US55", - "WY": "04000US56"} - -states = {"Alabama": "AL", "Alaska": "AK", "Arizona": "AZ", "Arkansas": "AR", "California": "CA", "Colorado": "CO", - "Connecticut": "CT", "District of Columbia": "DC", "Delaware": "DE", "Florida": "FL", "Georgia": "GA", - "Hawaii": "HI", "Idaho": "ID", "Illinois": "IL", "Indiana": "IN", "Iowa": "IA", "Kansas": "KS", - "Kentucky": "KY", "Louisiana": "LA", "Maine": "ME", "Maryland": "MD", "Massachusetts": "MA", "Michigan": "MI", - "Minnesota": "MN", "Mississippi": "MS", "Missouri": "MO", "Montana": "MT", "Nebraska": "NE", "Nevada": "NV", - "New Hampshire": "NH", "New Jersey": "NJ", "New Mexico": "NM", "New York": "NY", "North Carolina": "NC", - "North Dakota": "ND", "Ohio": "OH", "Oklahoma": "OK", "Oregon": "OR", "Pennsylvania": "PA", - "Rhode Island": "RI", "South Carolina": "SC", "South Dakota": "SD", "Tennessee": "TN", "Texas": "TX", - "Utah": "UT", "Vermont": "VT", "Virginia": "VA", "Washington": "WA", "West Virginia": "WV", - "Wisconsin": "WI", "Wyoming": "WY", "Chicago": "IL"} - -today = dt.date.today() -start_day = dt.date(2020, 1, 22) -diff = today - start_day -days = diff.days -dates = pd.date_range("2020-01-22", periods=days+1, - freq="D").strftime('%m-%d-%Y') -dates = pd.Series(dates).astype(str) - -data = [] -for date in dates: - - date_ = pd.to_datetime(date) - - try: - - url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/{}.csv".format( - date) - df = pd.read_csv(url, sep=",") - - if date_ <= pd.to_datetime("2020-03-21"): - - for coordinate in ["Latitude", "Longitude"]: - if coordinate not in list(df): - df[coordinate] = np.nan - - df = df[["Province/State", "Country/Region", "Confirmed", - "Deaths", "Recovered", "Latitude", "Longitude"]] - df = df.rename( - columns={"Country/Region": "Geography", "Province/State": "State"}) - - else: - - df = df[["Province_State", "Country_Region", - "Confirmed", "Deaths", "Recovered", "Lat", "Long_"]] - df = df.rename(columns={"Country_Region": "Geography", - "Province_State": "State", "Lat": "Latitude", "Long_": "Longitude"}) - - df["Date"] = date - df["Date"] = df["Date"].str[6:10] + "/" + \ - df["Date"].str[0:2] + "/" + df["Date"].str[3:5] - df = df[df["Geography"] == "US"] - df[["Confirmed", "Deaths", "Recovered"]] = df[[ - "Confirmed", "Deaths", "Recovered"]].astype("Int64") - df["Geography"] = df["Geography"].replace("US", "United States") - - data.append(df) - - except Exception as ex: - print(date, ex) - -data = pd.concat(data, sort=False) - -# To get some states Id's -new = data["State"].str.split(",", n=1, expand=True) -data["Place"] = new[0] -data["State_id"] = new[1] - -data["Place"] = data["Place"].replace(states) - -data["State_id"].fillna(value=pd.np.nan, inplace=True) -data["State_id"] = data["State_id"].fillna(0) - -data["ID Geography"] = data.apply( - lambda x: x["Place"] if x["State_id"] == 0 else x["State_id"], axis=1) -data["ID Geography"] = data["ID Geography"].str.strip() -data["ID Geography"] = data.apply( - lambda x: x["ID Geography"].replace(" (From Diamond Princess)", ""), axis=1) -data["ID Geography"] = data["ID Geography"].str.replace("D.C.", "WA") - -data = data.loc[~data["ID Geography"].isin(["Unassigned Location", "Grand Princess Cruise Ship", "Diamond Princess", - "U.S.", "Virgin Islands", "United States Virgin Islands", "Wuhan Evacuee", - "Recovered", "Grand Princess", "Puerto Rico", "Guam", "US", "American Samoa", - "Northern Mariana Islands"])] - -data = data.drop(columns={"State", "Place", "State_id"}) - -data["ID Geography"] = data["ID Geography"].replace(stateToDivision) - - -path = os.path.dirname(os.path.abspath("__file__")) + "/static/usacovid19.json" - -previous = pd.read_json(path) if os.path.exists(path) else pd.DataFrame([]) -if len(data) > len(previous): - data.to_json(path, orient="records")