Fixes #37929 - support download policies for file (#11184)

* Fixes #37929 - support download policy for file

* Cleanup unused method

* fix file tests

* Re-record VCRs

app/controllers/katello/api/v2/repositories_controller.rb

@@ -52,7 +52,7 @@ def custom_index_relation(collection)
       param :docker_upstream_name, String, :desc => N_("Name of the upstream docker repository")
       param :include_tags, Array, :desc => N_("Comma-separated list of tags to sync for a container image repository")
       param :exclude_tags, Array, :desc => N_("Comma-separated list of tags to exclude when syncing a container image repository. Default: any tag ending in \"-source\"")
-      param :download_policy, ["immediate", "on_demand"], :desc => N_("download policy for yum, deb, and docker repos (either 'immediate' or 'on_demand')")
+      param :download_policy, ["immediate", "on_demand"], :desc => N_("download policy for deb, docker, file and yum repos (either 'immediate' or 'on_demand')")
       param :download_concurrency, :number, :desc => N_("Used to determine download concurrency of the repository in pulp3. Use value less than 20. Defaults to 10")
       param :mirroring_policy, Katello::RootRepository::MIRRORING_POLICIES, :desc => N_("Policy to set for mirroring content.  Must be one of %s.") % RootRepository::MIRRORING_POLICIES
       param :verify_ssl_on_sync, :bool, :desc => N_("if true, Katello will verify the upstream url's SSL certifcates are signed by a trusted CA")

app/models/katello/root_repository.rb

@@ -23,7 +23,7 @@ class RootRepository < Katello::Model
     SKIPABLE_METADATA_TYPES = [Repository::YUM_TYPE, Repository::DEB_TYPE].freeze
 
     CONTENT_ATTRIBUTE_RESTRICTIONS = {
-      :download_policy => [Repository::YUM_TYPE, Repository::DEB_TYPE, Repository::DOCKER_TYPE],
+      :download_policy => [Repository::YUM_TYPE, Repository::DEB_TYPE, Repository::DOCKER_TYPE, Repository::FILE_TYPE],
     }.freeze
 
     MAX_EXPIRE_TIME = 7 * 24 * 60 * 60
@@ -218,12 +218,6 @@ def self.hosts_with_applicability
       ::Host.joins(:content_facet => :bound_repositories).where("#{Katello::Repository.table_name}.root_id" => self.select(:id))
     end
 
-    def ensure_no_download_policy
-      if !yum? && download_policy.present?
-        errors.add(:download_policy, _("cannot be set for non-yum repositories."))
-      end
-    end
-
     def ensure_no_checksum_on_demand
       if checksum_type.present? && download_policy == DOWNLOAD_ON_DEMAND
         errors.add(:checksum_type, _("Checksum type cannot be set for yum repositories with on demand download policy."))

app/services/katello/pulp3/repository/file.rb

@@ -17,12 +17,14 @@ def distribution_options(path)
         end
 
         def remote_options
+          options = common_remote_options
           #TODO: move to user specifying PULP_MANIFEST
           if root.url.blank?
-            common_remote_options.merge(url: nil)
+            options[:url] = nil
           else
-            common_remote_options.merge(url: root.url + '/PULP_MANIFEST')
+            options[:url] = root.url + '/PULP_MANIFEST'
           end
+          options.merge!(policy: root.download_policy)
         end
 
         def partial_repo_path

engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/details/views/repository-info.html

@@ -54,7 +54,7 @@ <h4 translate>Publishing Settings</h4>
           </dd>
 
           <dt ng-if="product.redhat != true" translate>Metadata Expiration (Seconds)</dt>
-          <dd bst-edit-number="repository.metadata_expire" on-save="save(repository)" 
+          <dd bst-edit-number="repository.metadata_expire" on-save="save(repository)"
               readonly="product.redhat || denied('edit_products', product) || !repository.allow_updates || repository.is_container_push"
               deletable="repository.metadata_expire !== null"
               on-delete="clearMetadataExpire()"
@@ -77,7 +77,7 @@ <h4 translate>Sync Settings</h4>
       </dd>
 
       <span ng-show="repository.generic_remote_options !== []">
-        <span ng-repeat="option in genericRemoteOptions"> 
+        <span ng-repeat="option in genericRemoteOptions">
           <dt>{{option.title}}</dt>
             <dd bst-edit-text="genericRemoteOptions[$index].value"
                 on-save="save(repository)"
@@ -333,7 +333,7 @@ <h4 translate>Sync Settings</h4>
           options="certs()"
           on-save="save(repository)">
       </dd>
-      <span ng-if="repository.content_type == 'yum' || repository.content_type == 'deb' || repository.content_type == 'docker'">
+      <span ng-if="repository.content_type === 'deb' || repository.content_type === 'docker' || repository.content_type === 'file' || repository.content_type === 'yum'">
         <dt translate>Download Policy</dt>
         <dd bst-edit-select="downloadPolicyDisplay(repository.download_policy)"
             readonly="denied('edit_products', product) || !repository.allow_updates || repository.is_container_push"

engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/new/new-repository.controller.js

@@ -143,7 +143,7 @@ angular.module('Bastion.repositories').controller('NewRepositoryController',
                         repository.ignorable_content.push("treeinfo");
                     }
                 }
-                if (repository.content_type !== 'yum' && repository.content_type !== 'deb' && repository.content_type !== 'docker') {
+                if (repository.content_type !== 'deb' && repository.content_type !== 'docker' && repository.content_type !== 'file' && repository.content_type !== 'yum') {
                     repository['download_policy'] = '';
                 }
                 if (repository.arch === 'No restriction') {

engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/new/views/new-repository.html

@@ -152,7 +152,7 @@ <h4 translate> Sync Settings </h4>
                 <br />
                 <div class="alert alert-danger" ><strong>Warning. </strong>Are you sure you want to add a comma in this whitespace separated list of distributions?</div>
        </div>
-     
+
       </div>
 
       <div ng-show="repository.content_type === 'deb'" bst-form-group label="{{ 'Components' | translate }}">
@@ -296,7 +296,7 @@ <h4 translate> Sync Settings </h4>
         </div>
       </div>
 
-      <div bst-form-group label="{{ 'Download Policy' | translate }}" ng-if="repository.content_type === 'yum' || repository.content_type === 'deb' || repository.content_type === 'docker'">
+      <div bst-form-group label="{{ 'Download Policy' | translate }}" ng-if="repository.content_type === 'deb' || repository.content_type === 'docker' || repository.content_type === 'file' || repository.content_type === 'yum'">
         <select id="download_policy"
                 name="download_policy"
                 ng-model="repository.download_policy"

test/actions/pulp3/orchestration/file_update_test.rb

@@ -59,6 +59,19 @@ def test_update_unset_unprotected
       assert_equal 1, dist_refs.count, "Expected 1 distribution reference."
     end
 
+    def test_update_policy
+      @repo.root.update(
+        download_policy: 'on_demand')
+
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Update,
+        @repo,
+        @primary)
+
+      file_remote = ::Katello::Pulp3::Api::File.new(@primary).remotes_api
+      assert_equal file_remote.list.results.find { |remote| remote.name == "Default_Organization-Cabinet-My_Files" }.policy, "on_demand"
+    end
+
     def test_update_set_unprotected
       @repo.root.update(unprotected: false)
 

test/fixtures/models/katello_root_repositories.yml

@@ -58,9 +58,9 @@ fedora_17_x86_64_root:
   label:                fedora_17_x86_64_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  url:                 "http://myrepo.com"
-  unprotected:           <%= true %>
-  download_policy: on_demand
+  url:                  "http://myrepo.com"
+  unprotected:          <%= true %>
+  download_policy:      on_demand
   mirroring_policy:     "mirror_content_only"
 
 fedora_17_x86_64_duplicate_root:
@@ -70,9 +70,9 @@ fedora_17_x86_64_duplicate_root:
   label:                fedora_17_x86_64_duplicate_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  url:                 "http://myrepo.com"
-  unprotected:           <%= true %>
-  download_policy: immediate
+  url:                  "http://myrepo.com"
+  unprotected:          <%= true %>
+  download_policy:      immediate
   mirroring_policy:     "mirror_content_only"
 
 feedless_fedora_17_x86_64_root:
@@ -82,8 +82,8 @@ feedless_fedora_17_x86_64_root:
   label:                feedless_fedora_17_x86_64_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  download_policy: on_demand
-  unprotected:           <%= true %>
+  download_policy:      on_demand
+  unprotected:          <%= true %>
   mirroring_policy:     "mirror_content_only"
 
 rhel_7_x86_64_root:
@@ -96,8 +96,8 @@ rhel_7_x86_64_root:
   label:                rhel_7_x86_64_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:redhat) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  url:                 'https://cdn.example.com/rhel/7/os'
-  download_policy: on_demand
+  url:                  'https://cdn.example.com/rhel/7/os'
+  download_policy:      on_demand
   mirroring_policy:     "mirror_content_only"
 
 rhel_6_x86_64_root:
@@ -110,8 +110,8 @@ rhel_6_x86_64_root:
   label:                rhel_6_x86_64_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:redhat) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  url:                 'https://cdn.example.com/rhel/6/os'
-  download_policy: on_demand
+  url:                  'https://cdn.example.com/rhel/6/os'
+  download_policy:      on_demand
   mirroring_policy:     "mirror_content_only"
 
 fedora_17_no_arch_root:
@@ -121,9 +121,9 @@ fedora_17_no_arch_root:
   label:                fedora_17_no_arch_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  url:                 "http://myrepo.com"
-  unprotected:           <%= true %>
-  download_policy: on_demand
+  url:                  "http://myrepo.com"
+  unprotected:          <%= true %>
+  download_policy:      on_demand
   mirroring_policy:     "mirror_content_only"
 
 rhel_7_no_arch_root:
@@ -136,8 +136,8 @@ rhel_7_no_arch_root:
   label:                rhel_7_no_arch_label
   product_id:           <%= ActiveRecord::FixtureSet.identify(:redhat) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  url:                 'https://cdn.example.com/rhel/7/os'
-  download_policy: on_demand
+  url:                  'https://cdn.example.com/rhel/7/os'
+  download_policy:      on_demand
   mirroring_policy:     "mirror_content_only"
 
 fedora_17_unpublished_root:
@@ -147,7 +147,7 @@ fedora_17_unpublished_root:
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
   url:                  "http://www.pleaseack.com"
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   download_policy:      immediate
   mirroring_policy:     "mirror_content_only"
 
@@ -158,8 +158,8 @@ fedora_17_unpublished_2_root:
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
   url:                  "http://www.pleaseack.com"
-  unprotected:           <%= true %>
-  download_policy: on_demand
+  unprotected:          <%= true %>
+  download_policy:      on_demand
   mirroring_policy:     "mirror_content_only"
 
 srpm_repo_root:
@@ -168,8 +168,8 @@ srpm_repo_root:
   content_type:         yum
   label:                source_rpm
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
-  download_policy: on_demand
-  unprotected:           <%= true %>
+  download_policy:      on_demand
+  unprotected:          <%= true %>
   mirroring_policy:     "mirror_content_only"
 
 redis_root:
@@ -181,7 +181,7 @@ redis_root:
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
   unprotected:           <%= true %>
   mirroring_policy:     "mirror_content_only"
-  download_policy:               immediate
+  download_policy:      immediate
 
 busybox_root:
   name:                 busybox
@@ -190,11 +190,11 @@ busybox_root:
   label:                busybox
   product_id:           <%= ActiveRecord::FixtureSet.identify(:puppet_product) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   url:                  'https://quay.io'
   docker_upstream_name: 'libpod/busybox'
   mirroring_policy:     "mirror_content_only"
-  download_policy:               immediate
+  download_policy:      immediate
 
 busybox2_root:
   name:                 busybox
@@ -203,11 +203,11 @@ busybox2_root:
   label:                busybox2
   product_id:           <%= ActiveRecord::FixtureSet.identify(:puppet_product) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   url:                  'https://quay.io'
   docker_upstream_name: 'libpod/busybox'
   mirroring_policy:     "mirror_content_only"
-  download_policy:               immediate
+  download_policy:      immediate
 
 generic_file_root:
   name:                 My Files
@@ -216,8 +216,9 @@ generic_file_root:
   label:                My_Files
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   mirroring_policy:     "mirror_content_only"
+  download_policy:      immediate
 
 pulp3_file_root_1:
   name:                 Pulp3 File 1
@@ -229,6 +230,7 @@ pulp3_file_root_1:
   unprotected:           <%= true %>
   url:                  "file:///var/lib/pulp/sync_imports/test_repos/file1"
   mirroring_policy:     "mirror_content_only"
+  download_policy:      immediate
 
 pulp3_ansible_collection_root_1:
   name:                 Pulp3 Ansible Collection 1
@@ -237,14 +239,14 @@ pulp3_ansible_collection_root_1:
   label:                Pulp3_Ansible_Collection_1
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   url:                  "https://galaxy.ansible.com/"
   ansible_collection_requirements: "---\n
   collections:\n
   - robertdebock.rundeck_collection"
   ansible_collection_auth_url: "https://some.authUrl.com"
   ansible_collection_auth_token: "random_token"
-  verify_ssl_on_sync: <%= false %>
+  verify_ssl_on_sync:   <%= false %>
   mirroring_policy:     "mirror_content_only"
 
 pulp3_docker_root_1:
@@ -254,19 +256,19 @@ pulp3_docker_root_1:
   label:                Pulp3_Docker_1
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           null
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   url:                  "https://registry-1.docker.io/"
   docker_upstream_name: "fedora/ssh"
   mirroring_policy:     "mirror_content_only"
-  download_policy:               immediate
+  download_policy:      immediate
 
 pulp3_deb_root_1:
   name:                 Pulp3 Deb 1
   content_id:           11
   content_type:         deb
   label:                Pulp3_Deb_1
   product_id:           <%= ActiveRecord::FixtureSet.identify(:debian) %>
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   url:                  "http://ftp.debian.mymirror.org/debian"
   deb_releases:         buster
   deb_components:       main
@@ -281,7 +283,7 @@ pulp3_python_root_1:
   label:                Pulp3_Python_1
   product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
   gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  unprotected:           <%= true %>
+  unprotected:          <%= true %>
   url:                  ""
   mirroring_policy:     "mirror_content_only"
 
@@ -298,15 +300,15 @@ uln_ovm2_2_1_1_i386_patch_root:
   upstream_password:    "baz"
 
 pulp3_ostree_root_1:
-  name: Pulp3 OSTree 1
-  content_id: 29
-  content_type: ostree
-  label: Pulp3_OSTree_1
-  product_id: <%= ActiveRecord::FixtureSet.identify(:fedora) %>
-  gpg_key_id: <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
-  unprotected: <%= true %>
-  url: ""
-  mirroring_policy: "mirror_content_only"
+  name:                 Pulp3 OSTree 1
+  content_id:           29
+  content_type:         ostree
+  label:                Pulp3_OSTree_1
+  product_id:           <%= ActiveRecord::FixtureSet.identify(:fedora) %>
+  gpg_key_id:           <%= ActiveRecord::FixtureSet.identify(:fedora_gpg_key) %>
+  unprotected:          <%= true %>
+  url:                  ""
+  mirroring_policy:     "mirror_content_only"
 
 product_host_count_repo1_root:
   name:                 product host count repo 1