Skip to content

Commit 2ff020d

Browse files
author
Vitalie D
committed
Active Scanning and Replacing (AWS) Tokens (#1265)
1 parent a92e03a commit 2ff020d

22 files changed

+779
-11
lines changed

‎Gemfile.lock

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ GEM
325325
rb-fsevent (0.9.8)
326326
rb-inotify (0.10.1)
327327
ffi (~> 1.0)
328-
rbtrace (0.4.12)
328+
rbtrace (0.4.14)
329329
ffi (>= 1.0.6)
330330
msgpack (>= 0.4.3)
331331
optimist (>= 3.0.0)

‎lib/travis/api/v3/models/scan_result.rb

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
2+
module Travis::API::V3
3+
class Models::ScanResult
4+
attr_reader :id, :log_id, :job_id, :owner_id, :owner_type, :created_at, :formatted_content, :issues_found, :archived, :purged_at, :token, :token_created_at,
5+
:job_number, :build_id, :build_number, :job_finished_at, :commit_sha, :commit_compare_url, :commit_branch, :repository_id
6+
7+
def initialize(attributes = {})
8+
@id = attributes.fetch('id')
9+
@log_id = attributes.fetch('log_id')
10+
@job_id = attributes.fetch('job_id')
11+
@owner_id = attributes.fetch('owner_id')
12+
@owner_type = attributes.fetch('owner_type')
13+
@created_at = attributes.fetch('created_at')
14+
@formatted_content = attributes.fetch('formatted_content')
15+
@issues_found = attributes.fetch('issues_found')
16+
@archived = attributes.fetch('archived')
17+
@purged_at = attributes.fetch('purged_at')
18+
@token = attributes.fetch('token')
19+
@token_created_at = attributes.fetch('token_created_at')
20+
@job_number = attributes.fetch('job_number')
21+
@build_id = attributes.fetch('build_id')
22+
@build_number = attributes.fetch('build_number')
23+
@job_finished_at = attributes.fetch('job_finished_at')
24+
@commit_sha = attributes.fetch('commit_sha')
25+
@commit_compare_url = attributes.fetch('commit_compare_url')
26+
@commit_branch = attributes.fetch('commit_branch')
27+
@repository_id = attributes.fetch('repository_id')
28+
end
29+
end
30+
end

‎lib/travis/api/v3/models/scanner_collection.rb

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# frozen_string_literal: true
2+
3+
module Travis::API::V3
4+
class Models::ScannerCollection
5+
def initialize(collection, total_count)
6+
@collection = collection
7+
@total_count = total_count
8+
end
9+
10+
def count(*)
11+
@total_count
12+
end
13+
14+
def limit(*)
15+
self
16+
end
17+
18+
def offset(*)
19+
self
20+
end
21+
22+
def map
23+
return @collection.map unless block_given?
24+
25+
@collection.map { |x| yield x }
26+
end
27+
28+
def to_sql
29+
"scanner_query:#{Time.now.to_i}"
30+
end
31+
end
32+
end

‎lib/travis/api/v3/permissions/repository.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ def create_request?
4242
write?
4343
end
4444

45+
def check_scan_results?
46+
write?
47+
end
48+
4549
def admin?
4650
access_control.adminable? object
4751
end

‎lib/travis/api/v3/queries/scan_result.rb

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
module Travis::API::V3
2+
class Queries::ScanResult < RemoteQuery
3+
params :id
4+
5+
def find
6+
scanner_client.get_scan_result(id)
7+
end
8+
9+
private
10+
11+
def scanner_client
12+
@_scanner_client ||= ScannerClient.new(nil)
13+
end
14+
end
15+
end

‎lib/travis/api/v3/queries/scan_results.rb

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
module Travis::API::V3
2+
class Queries::ScanResults < Query
3+
params :repository_id, :offset, :limit
4+
5+
def all
6+
# Reset the scan status on viewing the reports
7+
Repository.find(repository_id).update!(scan_failed_at: nil)
8+
9+
page = (offset.to_i / limit.to_i) + 1
10+
scanner_client(repository_id).scan_results(
11+
page.to_s,
12+
limit
13+
)
14+
end
15+
16+
private
17+
18+
def scanner_client(repository_id)
19+
@_scanner_client ||= ScannerClient.new(repository_id)
20+
end
21+
end
22+
end

‎lib/travis/api/v3/renderer/repository.rb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
module Travis::API::V3
22
class Renderer::Repository < ModelRenderer
33
representation(:minimal, :id, :name, :slug)
4-
representation(:standard, :id, :name, :slug, :description, :github_id, :vcs_id, :vcs_type, :github_language, :active, :private, :owner, :owner_name, :vcs_name, :default_branch, :starred, :managed_by_installation, :active_on_org, :migration_status, :history_migration_status, :shared, :config_validation, :server_type)
5-
representation(:experimental, :id, :name, :slug, :description, :vcs_id, :vcs_type, :github_id, :github_language, :active, :private, :owner, :default_branch, :starred, :current_build, :last_started_build, :next_build_number, :server_type)
6-
representation(:internal, :id, :name, :slug, :github_id, :vcs_id, :vcs_type, :active, :private, :owner, :default_branch, :private_key, :token, :user_settings, :server_type)
4+
representation(:standard, :id, :name, :slug, :description, :github_id, :vcs_id, :vcs_type, :github_language, :active, :private, :owner, :owner_name, :vcs_name, :default_branch, :starred, :managed_by_installation, :active_on_org, :migration_status, :history_migration_status, :shared, :config_validation, :server_type, :scan_failed_at)
5+
representation(:experimental, :id, :name, :slug, :description, :vcs_id, :vcs_type, :github_id, :github_language, :active, :private, :owner, :default_branch, :starred, :current_build, :last_started_build, :next_build_number, :server_type, :scan_failed_at)
6+
representation(:internal, :id, :name, :slug, :github_id, :vcs_id, :vcs_type, :active, :private, :owner, :default_branch, :private_key, :token, :user_settings, :server_type, :scan_failed_at)
77
representation(:additional, :allow_migration)
88

99
hidden_representations(:experimental, :internal)

‎lib/travis/api/v3/renderer/scan_result.rb

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
module Travis::API::V3
2+
class Renderer::ScanResult < ModelRenderer
3+
representation(:minimal, :id, :created_at, :formatted_content, :issues_found, :job_id, :build_id, :job_number, :build_number, :job_finished_at,
4+
:commit_sha, :commit_compare_url, :commit_branch, :build_created_by)
5+
representation(:standard, *representations[:minimal])
6+
7+
def build_created_by
8+
job = Travis::API::V3::Models::Job.find(model.job_id)
9+
build = Travis::API::V3::Models::Build.find(job.source_id)
10+
return nil unless creator = build.sender
11+
{
12+
'@type' => build.sender_type.downcase,
13+
'@href' => created_by_href(creator),
14+
'@representation' => 'minimal'.freeze,
15+
'id' => creator.id,
16+
'login' => creator.login
17+
}
18+
end
19+
20+
private def created_by_href(creator)
21+
case creator
22+
when V3::Models::Organization then Renderer.href(:organization, script_name: script_name, id: creator.id)
23+
when V3::Models::User then Renderer.href(:user, script_name: script_name, id: creator.id)
24+
end
25+
end
26+
end
27+
end

‎lib/travis/api/v3/renderer/scan_results.rb

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
module Travis::API::V3
2+
class Renderer::ScanResults < CollectionRenderer
3+
type :scan_results
4+
collection_key :scan_results
5+
end
6+
end

‎lib/travis/api/v3/routes.rb

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,16 @@ module Routes
283283
end
284284
end
285285

286+
resource :scan_results do
287+
route '/scan_results'
288+
get :all
289+
end
290+
291+
resource :scan_result do
292+
route '/scan_result/{scan_result.id}'
293+
get :find
294+
end
295+
286296
resource :user do
287297
capture id: :digit
288298
route '/user/{user.id}'

‎lib/travis/api/v3/scanner_client.rb

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
2+
# frozen_string_literal: true
3+
4+
module Travis::API::V3
5+
class ScannerClient
6+
class ConfigurationError < StandardError; end
7+
8+
def initialize(repository_id)
9+
@repository_id = repository_id
10+
end
11+
12+
def scan_results(page, limit)
13+
query_string = query_string_from_params(
14+
repository_id: @repository_id,
15+
limit: limit,
16+
page: page || '1',
17+
)
18+
response = connection.get("/scan_results?#{query_string}")
19+
20+
handle_errors_and_respond(response) do |body|
21+
scan_results = body['scan_results'].map do |scan_result|
22+
Travis::API::V3::Models::ScanResult.new(scan_result)
23+
end
24+
25+
Travis::API::V3::Models::ScannerCollection.new(scan_results, body.fetch('total_count', 0))
26+
end
27+
end
28+
29+
def get_scan_result(id)
30+
response = connection.get("/scan_results/#{id}")
31+
handle_errors_and_respond(response) do |body|
32+
Travis::API::V3::Models::ScanResult.new(body.fetch('scan_result'))
33+
end
34+
end
35+
36+
private
37+
38+
def handle_errors_and_respond(response)
39+
case response.status
40+
when 200, 201
41+
yield(response.body) if block_given?
42+
when 202
43+
true
44+
when 204
45+
true
46+
when 400
47+
raise Travis::API::V3::ClientError, response.body&.fetch('error', '')
48+
when 403
49+
raise Travis::API::V3::InsufficientAccess, response.body&.fetch('rejection_code', '')
50+
when 404
51+
raise Travis::API::V3::NotFound, response.body&.fetch('error', '')
52+
when 422
53+
raise Travis::API::V3::UnprocessableEntity, response.body&.fetch('error', '')
54+
else
55+
raise Travis::API::V3::ServerError, 'Scanner API failed'
56+
end
57+
end
58+
59+
def connection(timeout: 20)
60+
@connection ||= Faraday.new(url: scanner_url, ssl: { ca_path: '/usr/lib/ssl/certs' }) do |conn|
61+
conn.headers[:Authorization] = "Token token=\"#{scanner_token}\""
62+
conn.headers['Content-Type'] = 'application/json'
63+
conn.request :json
64+
conn.response :json
65+
conn.options[:open_timeout] = timeout
66+
conn.options[:timeout] = timeout
67+
conn.use OpenCensus::Trace::Integrations::FaradayMiddleware if Travis::Api::App::Middleware::OpenCensus.enabled?
68+
conn.adapter :net_http
69+
end
70+
end
71+
72+
def scanner_url
73+
Travis.config.scanner.url || raise(ConfigurationError, 'No Scanner API URL configured!')
74+
end
75+
76+
def scanner_token
77+
Travis.config.scanner.token || raise(ConfigurationError, 'No Scanner Auth Token configured!')
78+
end
79+
80+
def query_string_from_params(params)
81+
params.delete_if { |_, v| v.nil? || v.empty? }.to_query
82+
end
83+
end
84+
end

‎lib/travis/api/v3/services.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ module Services
3939
Leads = Module.new { extend Services }
4040
Lint = Module.new { extend Services }
4141
Log = Module.new { extend Services }
42+
ScanResult = Module.new { extend Services }
43+
ScanResults = Module.new { extend Services }
4244
Messages = Module.new { extend Services }
4345
Organization = Module.new { extend Services }
4446
Organizations = Module.new { extend Services }

‎lib/travis/api/v3/services/scan_result/find.rb

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
module Travis::API::V3
2+
class Services::ScanResult::Find < Service
3+
params :id
4+
5+
def run!
6+
raise LoginRequired unless access_control.full_access_or_logged_in?
7+
scan_result = query(:scan_result).find
8+
9+
repository = Travis::API::V3::Models::Repository.find(scan_result.repository_id)
10+
check_access(repository)
11+
12+
result scan_result
13+
end
14+
15+
def check_access(repository)
16+
access_control.permissions(repository).check_scan_results!
17+
end
18+
end
19+
end

‎lib/travis/api/v3/services/scan_results/all.rb

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
module Travis::API::V3
2+
class Services::ScanResults::All < Service
3+
params :repository_id
4+
paginate
5+
6+
def run!
7+
raise LoginRequired unless access_control.full_access_or_logged_in?
8+
9+
repository = Travis::API::V3::Models::Repository.find(params['repository_id'])
10+
check_access(repository)
11+
12+
result query(:scan_results).all
13+
end
14+
15+
def check_access(repository)
16+
access_control.permissions(repository).check_scan_results!
17+
end
18+
end
19+
end

‎lib/travis/config/defaults.rb

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,9 @@ def fallback_logs_api_auth_token
4141
auth: { target_origin: nil },
4242
assets: { host: HOSTS[Travis.env.to_sym] },
4343
amqp: { username: 'guest', password: 'guest', host: 'localhost', prefetch: 1 },
44-
billing: {},
4544
closeio: { key: 'key' },
4645
gdpr: {},
47-
insights: { endpoint: 'https://insights.travis-ci.dev/', auth_token: 'secret' },
4846
database: { adapter: 'postgresql', database: "travis_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } },
49-
fallback_logs_api: { url: fallback_logs_api_auth_url, token: fallback_logs_api_auth_token },
50-
logs_api: { url: logs_api_url, token: logs_api_auth_token },
5147
db: { max_statement_timeout_in_seconds: 15, slow_host_max_statement_timeout_in_seconds: 60},
5248
log_options: { s3: { access_key_id: '', secret_access_key: ''}},
5349
s3: { access_key_id: '', secret_access_key: ''},
@@ -86,10 +82,15 @@ def fallback_logs_api_auth_token
8682
build_backup_options: ENV['GCE_BUILD_BACKUP_OPTIONS'] ? JSON.parse(ENV['GCE_BUILD_BACKUP_OPTIONS']) : { gcs: { bucket_name: 'fillme', json_key: JSON.parse("{\n \"type\": \"service_account\",\n \"project_id\": \"fillme\",\n \"private_key_id\": \"b1c57117b4a0b8ae2af2f45b19a1cf9727bc6caf\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRaHA5z0vXNVSr\\nlLVd/smJpNkpzk4BoHq+zcuuzvKTf1ZY1LnrAhldUbDTKY67c06eOYwVrQc3tEIp\\nJCNhIDNY1lRfGJag6t2v5C710WY+X7qVnRhSpgthvWFX/Rm9KIv3be8AJvUTMUDQ\\nAL10eYrIWJOI/J59khuKvr7khIlysASwGoZc8UgcufxGuwCziNyEIfH1nxiBKILR\\nM1+LdYi/Avyb4bQth5x03THVEmdRDjV7Yoo2c17XElIXtkl03nUce4w8BCu+T1G2\\nKRkApHcQ8R0BDTjjBzaQuXTtTpLvkmZzJ1i0/kPNdnT70lv6N2nI1AN2DVkXDCkG\\nNlZ283W7AgMBAAECggEAC/W6CyMywqzSFCafISovGoRmvsOAowkmWYVpb6d0JUZt\\niQ9FOw3YowLKZZUHCN+yCslgndBPDDhoWu8schyjshwzn2bJG5GubaBLqlB2VXOk\\nNW1OeVHwbnmheKQE90+8hropn0maT6lNeVPBfkh+y6h7bKR47NUOa6MvRd/n9bvL\\nT7pP5ZAHoPoTcbUftOX0gDq0u+uRULe/rduxB0S2EHDEtZEH+ioUOP9AomnaRDSy\\n0spH1s2FUZxIbKBQzsrqMCai4MSjeUrJMTR3ZlpfXePirettvilSWqEXDLvvwaak\\ngehELuM5lH4T49wf4PmEYZ8Jqkh9ku+oNYJdJvGR2QKBgQD7KjhJx9usl0afrIH3\\nw7saHELluWGqHNa6j+TJDpY7N5lLLIym/br9d+cuLTF5CBEHJ502coDR9cyrLVZX\\na05CGmEfSVrSrLUyAU+mHHdsn8n6CCATmlgtPyzzt2c29J7dHUZL94zW/yG6Btg6\\nm+nY4eBKreLpj0+3KbhI/q0q1wKBgQDVcG8Ek3Kt2buOrpDBqxcwB31QljntT+7+\\nYcTZctYL/y7Lm2VcTjserNa3AjG59Z5iaQjKFPhbAvMHfppklyiVSVBRfn4bLTcx\\nSM9I+lntODtGI/BiHVE7hfoYKzwz/3Aj3npiOO9xnOfAgEubGn9DrOzLXPsvWN7E\\nz+/iSr4zvQKBgHFVB7kjGZizWgbKzIqEI3UQs479K3ibMrlUHKQslNV7rQwiugTQ\\nEQQ2inZnph866JQV5/adjEsxYn0LJB6mKNXjGVgIvZa6n7hEpzAJQEoff//2kqLF\\nzmv8SchfRY+iqdyUTRgSR9broMhUNlWb7NUUdyS7edxx8kJv7NvjLzhZAoGAMre9\\n2bOD26XSeKwof6y9HM+ayox4BVkqLE5lLVqpXD5uCznI0y9PwxFFEEW4NT0VPsNA\\nsGxdO5suzsgZve9hWGAMcuEA7EpJRC/N+cRrm//xrdAabeYTiHZkoFudualoJ03V\\nfQOUekXTmB2kWZ3pQdaUihp1IaIXhWL32Kj0G20CgYAfGInHUZf5mqxRK/id8B5Q\\nYmpnLDXsNu1I4qFKeaBo4dF2SGByNW7fVbK/BdCSg5Ov3Ui6m3QWBbJXh1a8QVYA\\nICvwJWqm53bNpocrFPAeXLy9xL5/5CEeVGQNcxFvUF3QgaPVmjbTsZk8vjbidUZk\\nOU8bArrUjGTxNJOe7GebhA==\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"[email protected]\",\n \"client_id\": \"100937792194965642651\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/service%40fillme.iam.gserviceaccount.com\"\n}\n") } },
8783
merge: { auth_token: 'merge-auth-token', api_url: 'https://merge.localhost' },
8884
force_authentication: false,
89-
yml: { url: 'https://yml.travis-ci.org', token: 'secret', auth_key: 'abc123' },
9085
read_only: ENV['READ_ONLY'] || false,
91-
vcs: {},
92-
job_log_access_permissions: { time_based_limit: false, access_based_limit: false, older_than_days: 365, max_days_value: 730, min_days_value: 30 }
86+
job_log_access_permissions: { time_based_limit: false, access_based_limit: false, older_than_days: 365, max_days_value: 730, min_days_value: 30 },
87+
billing: {},
88+
vcs: {},
89+
yml: { url: 'https://yml.travis-ci.org', token: 'secret', auth_key: 'abc123' },
90+
logs_api: { url: logs_api_url, token: logs_api_auth_token },
91+
fallback_logs_api: { url: fallback_logs_api_auth_url, token: fallback_logs_api_auth_token },
92+
scanner: {},
93+
insights: { endpoint: 'https://insights.travis-ci.dev/', auth_token: 'secret' }
9394

9495
default :_access => [:key]
9596

‎spec/spec_helper.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
require 'auth/helpers'
3232
require 'support/active_record'
3333
require 'support/billing_spec_helper'
34+
require 'support/scanner_spec_helper'
3435
require 'support/env'
3536
require 'support/formats'
3637
require 'support/gcs'
@@ -79,6 +80,7 @@ def parsed_body
7980
c.include Support::Env
8081
c.include Support::AuthHelpers, auth_helpers: true
8182
c.include Support::BillingSpecHelper, billing_spec_helper: true
83+
c.include Support::ScannerSpecHelper, scanner_spec_helper: true
8284
c.include Support::GdprSpecHelper, gdpr_spec_helper: true
8385

8486
# for auth tests against staging, how the hell does this work, if at all

0 commit comments

Comments
 (0)