Improved examples

- change S3 path (table moved from dev to PDS bucket s3://commoncrawl/)
- change monthly crawl in examples

Author: sebastian-nagel

README.md

@@ -49,18 +49,19 @@ First, the table needs to be imported into Athena:
 2. make Athena recognize the data partitions on `s3://`: `MSCK REPAIR TABLE 'ccindex';` (do not forget to adapt the table name). This step needs to be done again after new data partitions have been added.
 
 A couple of sample queries are also provided:
-- page/host/domain counts per top-level domain: [count-by-tld-page-host-domain.sql](src/sql/examples/count-by-tld-page-host-domain.sql)
+- count captures over partitions (crawls and subsets), get a quick overview how many pages are contained in the monthly crawl archives (and are also indexed in the table): [count-by-partition.sql](src/sql/examples/cc-index/count-by-partition.sql)
+- page/host/domain counts per top-level domain: [count-by-tld-page-host-domain.sql](src/sql/examples/cc-index/count-by-tld-page-host-domain.sql)
 - "word" count of
-  - host name elements (split host name at `.` into words): [count-hostname-elements.sql](src/sql/examples/count-hostname-elements.sql)
-  - URL path elements (separated by `/`): [count-url-path-elements.sql](src/sql/examples/count-url-path-elements.sql)
-- count HTTP status codes: [count-fetch-status.sql](src/sql/examples/count-fetch-status.sql)
-- count the domains of a specific top-level domain: [count-domains-of-tld.sql](src/sql/examples/count-domains-of-tld.sql)
-- compare document MIME types (Content-Type in HTTP response header vs. MIME type detected by [Tika](http://tika.apache.org/): [compare-mime-type-http-vs-detected.sql](src/sql/examples/compare-mime-type-http-vs-detected.sql)
-- distribution/histogram of host name lengths: [host_length_distrib.sql](src/sql/examples/host_length_distrib.sql)
-- count URL paths to robots.txt files [count-robotstxt-url-paths.sql](src/sql/examples/count-robotstxt-url-paths.sql)
+  - host name elements (split host name at `.` into words): [count-hostname-elements.sql](src/sql/examples/cc-index/count-hostname-elements.sql)
+  - URL path elements (separated by `/`): [count-url-path-elements.sql](src/sql/examples/cc-index/count-url-path-elements.sql)
+- count HTTP status codes: [count-fetch-status.sql](src/sql/examples/cc-index/count-fetch-status.sql)
+- count the domains of a specific top-level domain: [count-domains-of-tld.sql](src/sql/examples/cc-index/count-domains-of-tld.sql)
+- compare document MIME types (Content-Type in HTTP response header vs. MIME type detected by [Tika](http://tika.apache.org/): [compare-mime-type-http-vs-detected.sql](src/sql/examples/cc-index/compare-mime-type-http-vs-detected.sql)
+- distribution/histogram of host name lengths: [host_length_distrib.sql](src/sql/examples/cc-index/host_length_distrib.sql)
+- count URL paths to robots.txt files [count-robotstxt-url-paths.sql](src/sql/examples/cc-index/count-robotstxt-url-paths.sql)
 - export WARC record specs (file, offset, length) for
-  - a single domain: [get-records-of-domain.sql](src/sql/examples/get-records-of-domain.sql)
-  - a specific MIME type: [get-records-of-mime-type.sql](src/sql/examples/get-records-of-mime-type.sql)
-- find multi-lingual domains by analyzing URL paths: [get_language_translations_url_path.sql](src/sql/examples/get_language_translations_url_path.sql)
-- find similar domain names by Levenshtein distance (few characters changed): [similar-domains.sql](src/sql/examples/similar-domains.sql)
+  - a single domain: [get-records-of-domain.sql](src/sql/examples/cc-index/get-records-of-domain.sql)
+  - a specific MIME type: [get-records-of-mime-type.sql](src/sql/examples/cc-index/get-records-of-mime-type.sql)
+- find multi-lingual domains by analyzing URL paths: [get_language_translations_url_path.sql](src/sql/examples/cc-index/get_language_translations_url_path.sql)
+- find similar domain names by Levenshtein distance (few characters changed): [similar-domains.sql](src/sql/examples/cc-index/similar-domains.sql)
 

src/sql/athena/cc-index-create-table-flat.sql

@@ -5,10 +5,10 @@
 --  * format (STORED AS)
 --  * s3:// path (LOCATION)
 --
-CREATE EXTERNAL TABLE IF NOT EXISTS 'ccindex' (
-  url_surtkey   STRING,
-  url           STRING,
-  url_host_name STRING,
+CREATE EXTERNAL TABLE IF NOT EXISTS ccindex (
+  url_surtkey                   STRING,
+  url                           STRING,
+  url_host_name                 STRING,
   url_host_tld                  STRING,
   url_host_2nd_last_part        STRING,
   url_host_3rd_last_part        STRING,
@@ -18,20 +18,22 @@ CREATE EXTERNAL TABLE IF NOT EXISTS 'ccindex' (
   url_host_registered_domain    STRING,
   url_host_private_suffix       STRING,
   url_host_private_domain       STRING,
-  url_protocol  STRING,
-  url_port      INT,
-  url_path      STRING,
-  url_query     STRING,
-  fetch_time    TIMESTAMP,
-  fetch_status  SMALLINT,
-  content_digest    STRING,
-  content_mime_type STRING,
-  content_mime_detected STRING,
-  warc_filename         STRING,
-  warc_record_offset    INT,
-  warc_record_length    INT,
-  warc_segment          STRING)
-PARTITIONED BY(crawl STRING, subset STRING)
+  url_protocol                  STRING,
+  url_port                      INT,
+  url_path                      STRING,
+  url_query                     STRING,
+  fetch_time                    TIMESTAMP,
+  fetch_status                  SMALLINT,
+  content_digest                STRING,
+  content_mime_type             STRING,
+  content_mime_detected         STRING,
+  warc_filename                 STRING,
+  warc_record_offset            INT,
+  warc_record_length            INT,
+  warc_segment                  STRING)
+PARTITIONED BY (
+  crawl                         STRING,
+  subset                        STRING)
 STORED AS parquet
-LOCATION 's3://path_to_table/';
+LOCATION 's3://commoncrawl/cc-index/table/cc-main/warc/';
 

src/sql/athena/cc-index-create-table-nested.sql

@@ -5,7 +5,7 @@
 --  * format (STORED AS)
 --  * s3:// path (LOCATION)
 --
-CREATE EXTERNAL TABLE IF NOT EXISTS 'ccindex' (
+CREATE EXTERNAL TABLE IF NOT EXISTS ccindex (
   url     STRUCT<surtkey:STRING, url:STRING, host:STRUCT<name:STRING, tld:STRING, 2nd_last_part:STRING, 3rd_last_part:STRING, 4th_last_part:STRING, 5th_last_part:STRING, registry_suffix:STRING, registered_domain:STRING, private_suffix:STRING, private_domain:STRING>, protocol:STRING, port:INT, path:STRING, query:STRING>,
   fetch   STRUCT<time:TIMESTAMP, status:SMALLINT>,
   content STRUCT<digest:STRING, mime_type:STRING, mime_detected:STRING>,

src/sql/examples/compare-mime-type-http-vs-detected.sql renamed to src/sql/examples/cc-index/compare-mime-type-http-vs-detected.sql

@@ -5,7 +5,7 @@ SELECT COUNT(*) as n_pages,
        content_mime_type,
        content_mime_detected
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
 GROUP BY content_mime_type,
          content_mime_detected

src/sql/examples/cc-index/count-by-partition.sql

@@ -0,0 +1,8 @@
+-- count captures per partition (crawls and subsets):
+-- how many pages are contained in the monthly crawl
+-- archives (and are also indexed in the table)?
+SELECT COUNT(*) as n_captures,
+       crawl,
+       subset
+FROM "ccindex"."ccindex"
+GROUP BY crawl, subset;

src/sql/examples/count-by-tld-page-host-domain.sql renamed to src/sql/examples/cc-index/count-by-tld-page-host-domain.sql

@@ -4,7 +4,7 @@ SELECT COUNT(*) AS n_pages,
        COUNT(DISTINCT url_host_name) AS n_hosts,
        COUNT(DISTINCT url_host_registered_domain) AS n_domains
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
 GROUP BY  url_host_tld
 ORDER BY  n_pages DESC;

src/sql/examples/count-domains-of-tld.sql renamed to src/sql/examples/cc-index/count-domains-of-tld.sql

@@ -4,7 +4,7 @@
 SELECT COUNT(*) AS count,
        url_host_registered_domain
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND url_host_tld = 'no'
 GROUP BY  url_host_registered_domain

src/sql/examples/cc-index/count-fetch-status.sql

@@ -0,0 +1,10 @@
+-- fetch status counts
+-- (only for the 'crawldiagnostics' subset,
+--  captures in the 'warc' subset have status 200)
+SELECT COUNT(*) as count,
+       fetch_status
+FROM "ccindex"."ccindex"
+WHERE crawl = 'CC-MAIN-2018-05'
+  AND subset = 'crawldiagnostics'
+GROUP BY  fetch_status
+ORDER BY  count DESC;

src/sql/examples/count-hostname-elements.sql renamed to src/sql/examples/cc-index/count-hostname-elements.sql

@@ -10,7 +10,7 @@ SELECT host_name_part,
 FROM "ccindex"."ccindex",
      UNNEST(reverse(split(url_host_name, '.')))
         WITH ORDINALITY AS t (host_name_part, part_position)
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND url_host_tld = 'fr'
 GROUP BY host_name_part

src/sql/examples/count-robotstxt-url-paths.sql renamed to src/sql/examples/cc-index/count-robotstxt-url-paths.sql

@@ -4,7 +4,7 @@
 SELECT COUNT(*) AS count,
        URL_EXTRACT_PATH(url) AS urlpath
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'robotstxt'
 GROUP BY URL_EXTRACT_PATH(url)
 HAVING (COUNT(*) >= 100)
@@ -15,7 +15,7 @@ ORDER BY  count DESC
 --  SELECT COUNT(*) AS count,
 --         url_path
 --  FROM "ccindex"."ccindex"
---  WHERE crawl = 'CC-MAIN-2017-47'
+--  WHERE crawl = 'CC-MAIN-2018-05'
 --    AND subset = 'robotstxt'
 --  GROUP BY url_path
 --  HAVING (COUNT(*) >= 100)

src/sql/examples/count-url-path-elements.sql renamed to src/sql/examples/cc-index/count-url-path-elements.sql

@@ -7,7 +7,7 @@ SELECT url_path_element,
        COUNT(url_path_element) as frequency
 FROM "ccindex"."ccindex",
   UNNEST(transform(filter(split(url_path, '/'), w -> w != ''), w -> url_decode(w))) AS t (url_path_element)
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND url_host_tld = 'fr'
 GROUP BY url_path_element

src/sql/examples/get-language-translations-url-path.sql renamed to src/sql/examples/cc-index/get-language-translations-url-path.sql

@@ -12,10 +12,10 @@
 SELECT url_host_registered_domain,
        COUNT(DISTINCT(url_path_lang)) as n_lang,
        COUNT(*) as n_pages,
-       histogram(DISTINCT(url_path_lang)) as lang_counts
+       histogram(url_path_lang) as lang_counts
 FROM "ccindex"."ccindex",
   UNNEST(regexp_extract_all(url_path, '(?<=/)(?:[a-z][a-z])(?=/)')) AS t (url_path_lang)
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND url_host_registry_suffix = 'va'
 GROUP BY url_host_registered_domain

src/sql/examples/get-records-of-domain.sql renamed to src/sql/examples/cc-index/get-records-of-domain.sql

@@ -6,7 +6,7 @@ SELECT url,
        warc_record_offset,
        warc_record_length
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND url_host_registered_domain = 'commoncrawl.org'
 LIMIT 1000;

src/sql/examples/get-records-of-mime-type.sql renamed to src/sql/examples/cc-index/get-records-of-mime-type.sql

@@ -13,7 +13,7 @@ SELECT url,
        warc_record_offset,
        warc_record_length
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND (content_mime_detected = 'application/pdf' OR
        content_mime_type LIKE '%pdf%')

src/sql/examples/host_length_distrib.sql renamed to src/sql/examples/cc-index/host_length_distrib.sql

@@ -7,7 +7,7 @@
 WITH t AS (SELECT url_host_name AS host_name,
                   cardinality(split(url_host_name, '.')) AS host_name_length
            FROM "ccindex"."ccindex"
-           WHERE crawl = 'CC-MAIN-2017-47'
+           WHERE crawl = 'CC-MAIN-2018-05'
              AND subset = 'warc')
 SELECT COUNT(*) AS n_pages,
        COUNT(DISTINCT host_name) AS uniq_hosts,

src/sql/examples/similar-domains.sql renamed to src/sql/examples/cc-index/similar-domains.sql

@@ -10,7 +10,7 @@ SELECT COUNT(*) AS n_pages,
        url_host_2nd_last_part AS host_2nd_last_part,
        url_host_registered_domain AS domain
 FROM "ccindex"."ccindex"
-WHERE crawl = 'CC-MAIN-2017-47'
+WHERE crawl = 'CC-MAIN-2018-05'
   AND subset = 'warc'
   AND (levenshtein_distance('wikipedia', url_host_2nd_last_part) <= 2)
 GROUP BY url_host_tld, url_host_2nd_last_part, url_host_registered_domain