Porthos start RPM release on Tuesdays (#21)

nethesis/dev#5641

Author: DavidePrincipi

porthos/README.md

@@ -122,8 +122,7 @@ For instance to create a key on athos
 The `tier_id` value should be a number. If the value is negative, the tier
 number is calculated by an hash function, based on the system identifier. 
 
-If `tier_id` is not a number, both `auth.php` and `subscription.php` reply with
-403 - forbidden.
+If `tier_id` is not a number, `stable/` is served instead.
 
 ### `icat` field
 
@@ -150,19 +149,16 @@ from `/etc/porthos/repos.conf`. Upstream YUM rsync URLs are defined there.
 The following commands are executed automatically, as defined in `porthos.cron`:
 
 - `repo-bulk-pull` creates a snapshot date-dir (e.g. `d20180301`) under
-  `dest_dir` with differences from upstream repositories. It sets `t0` to point at
-  it.
-- `repo-bulk-shift [N]` updates `t1` ... `tN` links by shifting tiers of one position
-  the optional `N` parameter creates missing links up to N - 1.
+  `dest_dir` with differences from upstream repositories.
 - `repo-bulk-cleanup` erases stale snapshots directories
 
 The following commands are designed for Porthos initialization, to recover from errors, or implement low-level actions:
 
 - `repo-bulk-hinit` runs initial synchronization from upstream repositories (-f disables the check for already existing directories)
-- `repo-head-init`  initial/override synchronization of head from a specific upstream repo
-- `repo-head-rollback` roll back head to a previous snapshot for a specific repo
-- `repo-tier-pull`  create a new upstream snapshot for a specific repo
-- `repo-tier-delete`  delete repomd.xml from a given tier or snapshot
+- `repo-head-init`  synchronization of head from a specific upstream repo
+- `repo-head-rollback` roll back a repository head state to a previous snapshot state
+- `repo-snapshot-create` create a new repository snapshot
+- `repo-snapshot-delete` delete repomd.xml from a given repository snapshot
 - `repo-rpm-lookup`  seek the given RPM in every snapshot for a given repository
 - `xrsync` run rsync safely, trying to repeat the operation if it fails
 
@@ -179,8 +175,9 @@ it fails as soon as no snapshot is found, or if an invalid repository identifier
 is issued.
 
 Some times it is desirable to re-sync the head repository, without generating a
-new snapshot, like `repo-tier-pull` does. That happens if an upstream repo was
-fixed before being shifted. In that case run `repo-head-init` as follow:
+new snapshot, like `repo-snapshot-create` does. That happens if an upstream repo
+was fixed quickly and the bogus RPM never entered any snapshot. In that case run
+`repo-head-init` as follow:
 
     repo-head-init -n -f 7.6.1810/nethserver-updates/x86_64
 
@@ -190,19 +187,19 @@ command to run even if the repository was already initialized.
 If one or more snapshots contain a bogus RPM it is possible to delete the whole
 repository metadata (repomd.xml) file with the following command:
 
-    repo-tier-delete d20190702/7.6.1810/nethserver-updates/x86_64 d20190630/7.6.1810/nethserver-updates/x86_64
+    repo-snapshot-delete d20190702/7.6.1810/nethserver-updates/x86_64 d20190630/7.6.1810/nethserver-updates/x86_64
 
-The correct snapshot (or tier) name can be found starting from the RPM name with:
+The correct snapshot name can be found starting from the RPM name with:
 
     repo-rpm-lookup bogus-rpm-1.2.3-1.ns7.noarch.rpm
     d20190702/7.6.1810/nethserver-updates/x86_64
     d20190630/7.6.1810/nethserver-updates/x86_64
 
 The two commands can be combined together with `xargs`:
 
-    repo-rpm-lookup bogus-rpm-1.2.3-1.ns7.noarch.rpm | xargs -- repo-tier-delete
+    repo-rpm-lookup bogus-rpm-1.2.3-1.ns7.noarch.rpm | xargs -- repo-snapshot-delete
 
-If the RPM is found under `head/`, `repo-tier-delete` safely ignores it.
+If the RPM is found under `head/`, `repo-snapshot-delete` safely ignores it.
 
 ## Automated schedule
 

porthos/root/etc/cron.d/porthos.cron

@@ -5,9 +5,8 @@
 RANDOM=20
 CRON_TZ=UTC
 
-0 21 * * 1        root /usr/local/bin/repo-bulk-pull
-0 21 * * 2-4      root /usr/local/bin/repo-bulk-shift
-0 22 * * 1-4      root /usr/local/bin/repo-bulk-cleanup
+0 21 * * 0-4      root /usr/local/bin/repo-bulk-pull
+0 22 * * 0-4      root /usr/local/bin/repo-bulk-cleanup >/dev/null
 
 # end
 

porthos/root/etc/nginx/conf.d/porthos.conf

@@ -38,29 +38,9 @@ server {
         return 200 "pong\n";
     }
 
-    location ~ "^/T0/(.*)" {
-        internal;
-        try_files /t0/$1 /head/$1;
-    }
-
-    location ~ "^/T1/(.*)" {
-        internal;
-        try_files /t1/$1 /t0/$1 /head/$1;
-    }
-
-    location ~ "^/T2/(.*)" {
-        internal;
-        try_files /t2/$1 /t1/$1 /t0/$1 /head/$1;
-    }
-
-    location ~ "^/T3/(.*)" {
-        internal;
-        try_files /t3/$1 /t2/$1 /t1/$1 /t0/$1 /head/$1;
-    }
-
     # add another location for tier "Tx" here...
 
-    location ~ "^/(t[0-9]|d[0-9]{8}|head)(/.*)" {
+    location ~ "^/(d[0-9]{8}|head)(/.*)" {
         internal;
     }
 

porthos/root/srv/porthos/script/auth.php

@@ -22,6 +22,7 @@
 
 require_once("lib.php");
 require_once("config-" . $_SERVER['PORTHOS_SITE'] . ".php");
+ini_set('date.timezone', $config['timezone']);
 
 $uri = parse_uri($_SERVER['DOCUMENT_URI']);
 
@@ -54,30 +55,36 @@
 if($config['legacy_auth']) {
     $valid_credentials = $valid_credentials || $_SERVER['PHP_AUTH_USER'] ===  $_SERVER['PHP_AUTH_PW'];
 }
-$has_access_disabled = ! is_numeric($access['tier_id']);
 
 if($access['tier_id'] < 0) {
     $hash = 0;
     foreach(str_split($_SERVER['PHP_AUTH_USER']) as $c) {
         $hash += ord($c);
     }
     $hash = $hash % 256;
-    if($hash < 13) { // 5%
+    if($hash < 26) { // 10%
         $tier_id = 0;
-    } elseif($hash < 51) { // +15% = 20%
+    } elseif($hash < 77) { // +20% = 30%
         $tier_id = 1;
-    } elseif($hash < 128) { // +30% = 50%
+    } else { // +70% = 100%
         $tier_id = 2;
-    } else { // +50% = 100%
-        $tier_id = 3;
     }
-    $tier_id += $config['tier_id_base'];
 } else {
-    $tier_id = intval($access['tier_id']);
+    $tier_id = $access['tier_id'];
+}
+
+$is_tier_request = is_numeric($tier_id) && $uri['prefix'] == 'autoupdate';
+if($is_tier_request && $valid_credentials) {
+    // Seeking a snapshot is a time-consuming op. Ensure we have valid
+    // credentials before running it!
+    $snapshot = lookup_snapshot($uri['full_path'], $tier_id, $config['week_size']);
+} else {
+    $snapshot = 'head';
 }
 
 if(basename($uri['rest']) == 'repomd.xml') {
-    header('Cache-Control: private');
+    // repomd.xml is the entry point of repository (meta)data: let's keep track
+    // of every client access to repositories:
     application_log(json_encode(array(
         'porthos_site' => $_SERVER['PORTHOS_SITE'],
         'connection' => $_SERVER['CONNECTION'] ?: '',
@@ -86,19 +93,18 @@
         'repo' => $uri['repo'],
         'version' => $uri['version'],
         'arch' => $uri['arch'],
-        'tier_id' => $uri['prefix'] == 'autoupdate' ? $tier_id : NULL,
+        'tier_id' => $is_tier_request ? $tier_id : FALSE,
         'tier_auto' => isset($hash),
         'tls' => isset($_SERVER['HTTPS']),
-        'auth_response' => ! $valid_credentials ? 'bad_credentials' : $has_access_disabled ? 'bad_access' : 'pass',
+        'auth_response' => ! $valid_credentials ? 'bad_credentials' : 'pass',
+        'snapshot' => $snapshot,
     )));
 }
 
-if ($has_access_disabled || ! $valid_credentials) {
+if (! $valid_credentials) {
+    // Exit here, after sending the application_log record for repomd.xml requests.
     exit_http(403);
 }
 
-if($uri['prefix'] == 'autoupdate') {
-    return_file('/T' . $tier_id . $uri['full_path']);
-} else {
-    return_file('/head' . $uri['full_path']);
-}
+header('Cache-Control: private');
+return_file('/' . $snapshot . $uri['full_path']);

porthos/root/srv/porthos/script/config-porthos.php

@@ -47,8 +47,11 @@
 //     if FALSE, check the user name and the password separately
 $config['legacy_auth'] = FALSE;
 
-// tier_id_base (int)
-//     this is the base/minimum tier_id value, when automatic tier_id (-1) is set.
-//     - be sure that (tier_id_base + 3) < "number of tiers"
-//     - run repo-bulk-shift N (with N = "number of tiers") to add more tier links
-$config['tier_id_base'] = 0;
+// week_size (int)
+//     number of days/snapshots per week. Ensure this value is consistent with
+//     your crontab settings
+$config['week_size'] = 5;
+
+// timezone (string)
+//     the PHP timezone for this application
+$config['timezone'] = 'UTC';

porthos/root/srv/porthos/script/lib.php

@@ -70,3 +70,27 @@ function parse_uri($uri) {
     $parts = array_merge($parts, $matches);
     return $parts;
 }
+
+function get_snapshot_timestamp($snapshot_name) {
+    $parts = array();
+    if(!$snapshot_name || !preg_match('/^d(?<year>\d\d\d\d)(?<month>\d\d)(?<day>\d\d)$/', $snapshot_name, $parts)) {
+        return time();
+    }
+    return mktime(0, 0, 0, $parts['month'], $parts['day'], $parts['year']);
+}
+
+function lookup_snapshot($path, $tier_id = 0, $week_size = 5) {
+    $root_path = "/srv/porthos/webroot/";
+    $snapshots = array_reverse(array_map('basename', glob($root_path . "d20*")));
+    $last_snapshot_day_id = date('w', get_snapshot_timestamp($snapshots[0]));
+    // $monday_offset formula:
+    //     ($last_snapshot_day_id-1): rebase on Mondays
+    //     ($last_snapshot_day_id > $tier_id ? 0 : $week_size): select current week Monday or previous one
+    $monday_offset = ($last_snapshot_day_id-1) + ($last_snapshot_day_id > $tier_id ? 0 : $week_size);
+    for($i = min($monday_offset, count($snapshots) - 1); $i >= 0; $i--) {
+        if(is_file($root_path . $snapshots[$i] . '/' . $path)) {
+            break;
+        }
+    }
+    return $i < 0 ? 'head' : $snapshots[$i];
+}

porthos/root/srv/porthos/script/subscription.php

@@ -47,8 +47,7 @@
 if($config['legacy_auth']) {
     $valid_credentials = $valid_credentials || $_SERVER['PHP_AUTH_USER'] ===  $_SERVER['PHP_AUTH_PW'];
 }
-$has_access_disabled = ! is_numeric($access['tier_id']) || $access['icat'] === FALSE;
-if ($has_access_disabled || ! $valid_credentials) {
+if (! $valid_credentials) {
     exit_http(403);
 }
 

porthos/root/usr/local/bin/repo-bulk-cleanup

@@ -26,8 +26,8 @@ shopt -s nullglob extglob
 . /etc/porthos/repos.conf
 cd ${dest_dir}
 
-# 4 weeks before last repo-tier-pull:
-stale_ts=$[ $(stat -c %Y head) - (86400*28) ]
+# 2 weeks before last repo-bulk-pull:
+stale_ts=$[ $(stat -c %Y head) - (86400*14) ]
 
 declare -A targets
 
@@ -38,11 +38,6 @@ for D in d+([0-9]); do
     fi
 done
 
-# Always preserve a symlinked tier
-for T in t[0-9]; do
-    unset "targets[$(readlink ${T})]"
-done
-
 if [[ -n ${!targets[@]} ]]; then
     echo "[NOTICE] removing stale tier(s) ${!targets[@]}"
     rm -rf ${!targets[@]}

porthos/root/usr/local/bin/repo-bulk-pull

@@ -27,10 +27,7 @@ date_dir=d$(date +%Y%m%d)
 trap 'kill $(jobs -p)' INT HUP TERM
 
 for repo in ${!repos[@]}; do
-    /usr/local/bin/repo-tier-pull $* ${repo} ${date_dir} &
+    /usr/local/bin/repo-snapshot-create $* ${repo} ${date_dir} &
 done
 
 wait
-
-cd ${dest_dir}
-ln -sfT ${date_dir} t0