Skip to content

Commit a8372f8

Browse files
sschuberthsschuberth
committed
feat(package-managers): Start to implement a BitBake analyzer
Add the beginnings of an analyzer for BitBake [1], see [2] for context. It basically works by making the build inherit from the "create-spdx" class [3] to create SPDX 2.2 files and post-processing them via ORT's SPDX document file analyzer. This initial implementations has several known limitations still. First of all, as the "create-spdx" class cannot be used without building, and builds are not cached, so the analyzer is very slow. Secondly, SPDX external documents refs cannot be resolved yet. This requires some post-processing of the SPDX document files before passing them on, notably by adjusting the `SPDX_NAMESPACE_PREFIX` variable [4]. [1]: https://docs.yoctoproject.org/bitbake/ [2]: #722 [3]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html [4]: https://docs.yoctoproject.org/ref-manual/variables.html#term-SPDX_NAMESPACE_PREFIX Signed-off-by: Sebastian Schuberth <[email protected]>
1 parent 508dc81 commit a8372f8

File tree

9 files changed

+398
-0
lines changed

9 files changed

+398
-0
lines changed

‎analyzer/src/funTest/kotlin/PackageManagerFunTest.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import org.ossreviewtoolkit.model.config.PathExcludeReason
3939
class PackageManagerFunTest : WordSpec({
4040
val definitionFiles = listOf(
4141
"bazel/MODULE.bazel",
42+
"bitbake/recipe.bb",
4243
"bower/bower.json",
4344
"bundler/Gemfile",
4445
"cargo/Cargo.toml",

‎plugins/package-managers/bitbake/README.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# About
2+
3+
This is a package manager plugin for the [OSS Review Toolkit][ORT] to analyze [Yocto] projects managed by [BitBake].
4+
It supersedes the combination of the [meta-doubleopen] and [do-convert] projects by relying on upstream [SBOM] generation in [SPDX] format, and converting the generated files to an ORT analyzer result file via ORT's [SPDX document file analyzer].
5+
6+
[ORT]: https://github.com/oss-review-toolkit/ort
7+
[BitBake]: https://docs.yoctoproject.org/bitbake.html
8+
[Yocto]: https://www.yoctoproject.org/
9+
[meta-doubleopen]: https://github.com/doubleopen-project/meta-doubleopen
10+
[do-convert]: https://github.com/doubleopen-project/do-convert
11+
[SBOM]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
12+
[SPDX]: https://spdx.dev/
13+
[SPDX document file analyzer]: https://oss-review-toolkit.org/ort/docs/tools/analyzer

‎plugins/package-managers/bitbake/build.gradle.kts

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
plugins {
21+
// Apply precompiled plugins.
22+
id("ort-library-conventions")
23+
}
24+
25+
dependencies {
26+
api(project(":analyzer"))
27+
api(project(":model"))
28+
29+
implementation(project(":utils:common-utils"))
30+
implementation(project(":plugins:package-managers:spdx-package-manager"))
31+
32+
funTestImplementation(project(":downloader"))
33+
funTestImplementation(project(":plugins:version-control-systems:git-version-control-system"))
34+
funTestImplementation(testFixtures(project(":analyzer")))
35+
}

‎plugins/package-managers/bitbake/src/funTest/kotlin/BitBakeToolFunTest.kt

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
package org.ossreviewtoolkit.plugins.packagemanagers.bitbake
21+
22+
import io.kotest.core.spec.style.WordSpec
23+
import io.kotest.engine.spec.tempdir
24+
import io.kotest.matchers.collections.beEmpty
25+
import io.kotest.matchers.collections.shouldHaveSize
26+
import io.kotest.matchers.result.shouldBeSuccess
27+
import io.kotest.matchers.should
28+
import io.kotest.matchers.shouldBe
29+
import io.kotest.matchers.string.shouldMatch
30+
31+
import org.ossreviewtoolkit.analyzer.Analyzer
32+
import org.ossreviewtoolkit.analyzer.create
33+
import org.ossreviewtoolkit.model.Identifier
34+
import org.ossreviewtoolkit.model.VcsInfo
35+
import org.ossreviewtoolkit.model.VcsType
36+
import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
37+
import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
38+
import org.ossreviewtoolkit.utils.test.ExpensiveTag
39+
import org.ossreviewtoolkit.utils.test.shouldNotBeNull
40+
41+
class BitBakeToolFunTest : WordSpec({
42+
"BitBake" should {
43+
"get the version correctly" {
44+
val bitBake = create("BitBake") as BitBake
45+
46+
val version = bitBake.getBitBakeVersion(tempdir())
47+
48+
version shouldMatch "\\d+\\.\\d+\\.\\d+"
49+
}
50+
}
51+
52+
"Analyzing recipes from Poky" should {
53+
val projectDir = tempdir()
54+
val pokyVcsInfo = VcsInfo(VcsType.GIT, "https://git.yoctoproject.org/poky", "kirkstone-4.0.17")
55+
56+
Git().run {
57+
val workingTree = initWorkingTree(projectDir, pokyVcsInfo)
58+
updateWorkingTree(workingTree, pokyVcsInfo.revision)
59+
} shouldBeSuccess pokyVcsInfo.revision
60+
61+
"create an SPDX file for the 'quilt-native' package" {
62+
val recipeFileName = "quilt-native_0.67.bb"
63+
val result = Analyzer(AnalyzerConfiguration()).run {
64+
val fileInfo = findManagedFiles(projectDir)
65+
val singleFileInfo = fileInfo.copy(
66+
managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) ->
67+
packageManager to definitionsFiles.filter { it.name == recipeFileName }
68+
}.toMap()
69+
)
70+
analyze(singleFileInfo)
71+
}
72+
73+
result.analyzer?.result shouldNotBeNull {
74+
projects shouldHaveSize 1
75+
76+
with(projects.single()) {
77+
id shouldBe Identifier("BitBake:OpenEmbedded ():quilt-native:0.67")
78+
declaredLicenses shouldBe setOf("GPL-2.0-only")
79+
homepageUrl shouldBe "http://savannah.nongnu.org/projects/quilt/"
80+
scopes should beEmpty()
81+
}
82+
}
83+
}
84+
85+
"create a SPDX files for the 'xmlto' package".config(tags = setOf(ExpensiveTag)) {
86+
val recipeFileName = "xmlto_0.0.28.bb"
87+
val result = Analyzer(AnalyzerConfiguration()).run {
88+
val fileInfo = findManagedFiles(projectDir)
89+
val singleFileInfo = fileInfo.copy(
90+
managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) ->
91+
packageManager to definitionsFiles.filter { it.name == recipeFileName }
92+
}.toMap()
93+
)
94+
analyze(singleFileInfo)
95+
}
96+
97+
result.analyzer?.result shouldNotBeNull {
98+
projects shouldHaveSize 90
99+
}
100+
}
101+
}
102+
})

‎plugins/package-managers/bitbake/src/main/kotlin/BitBake.kt

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
package org.ossreviewtoolkit.plugins.packagemanagers.bitbake
21+
22+
import java.io.File
23+
24+
import kotlin.time.measureTime
25+
26+
import org.apache.logging.log4j.kotlin.logger
27+
28+
import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
29+
import org.ossreviewtoolkit.analyzer.PackageManager
30+
import org.ossreviewtoolkit.analyzer.PackageManagerResult
31+
import org.ossreviewtoolkit.model.ProjectAnalyzerResult
32+
import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
33+
import org.ossreviewtoolkit.model.config.RepositoryConfiguration
34+
import org.ossreviewtoolkit.plugins.packagemanagers.spdx.SpdxDocumentFile
35+
import org.ossreviewtoolkit.utils.common.ProcessCapture
36+
import org.ossreviewtoolkit.utils.common.getCommonParentFile
37+
import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
38+
import org.ossreviewtoolkit.utils.common.withoutPrefix
39+
import org.ossreviewtoolkit.utils.ort.createOrtTempDir
40+
import org.ossreviewtoolkit.utils.ort.createOrtTempFile
41+
42+
/**
43+
* A package manager that uses OpenEmbedded's "bitbake" tool to create SPDX SBOMs [1][2] e.g. for Yocto distributions,
44+
* and post-processes these into ORT analyzer results.
45+
*
46+
* [1]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
47+
* [2]: https://dev.to/angrymane/create-spdx-with-yocto-2od9
48+
*/
49+
class BitBake(
50+
name: String,
51+
analysisRoot: File,
52+
analyzerConfig: AnalyzerConfiguration,
53+
repoConfig: RepositoryConfiguration
54+
) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig) {
55+
class Factory : AbstractPackageManagerFactory<BitBake>("BitBake") {
56+
override val globsForDefinitionFiles = listOf("*.bb")
57+
58+
override fun create(
59+
analysisRoot: File,
60+
analyzerConfig: AnalyzerConfiguration,
61+
repoConfig: RepositoryConfiguration
62+
) = BitBake(type, analysisRoot, analyzerConfig, repoConfig)
63+
}
64+
65+
private val scriptFile by lazy { extractResourceToTempFile(BITBAKE_SCRIPT_NAME).apply { setExecutable(true) } }
66+
private val spdxConfFile by lazy { extractResourceToTempFile(SPDX_CONF_NAME) }
67+
68+
private val spdxManager by lazy { SpdxDocumentFile(name, analysisRoot, analyzerConfig, repoConfig) }
69+
70+
override fun resolveDependencies(definitionFiles: List<File>, labels: Map<String, String>): PackageManagerResult {
71+
val commonDefinitionDir = getCommonParentFile(definitionFiles)
72+
val workingDir = requireNotNull(commonDefinitionDir.searchUpwardsForFile(INIT_SCRIPT_NAME)) {
73+
"No '$INIT_SCRIPT_NAME' script file found for directory '$commonDefinitionDir'."
74+
}
75+
76+
logger.info { "Determined the working directory to be '$workingDir'." }
77+
78+
val localVersion = getBitBakeVersion(workingDir)
79+
val globalVersion = createOrtTempDir().let { dir ->
80+
getBitBakeVersion(dir).also { dir.safeDeleteRecursively(force = true) }
81+
}
82+
83+
if (localVersion != globalVersion) {
84+
logger.warn { "Local $managerName version $localVersion differs from global version $globalVersion." }
85+
}
86+
87+
val deployDirs = mutableSetOf<File>()
88+
89+
definitionFiles.forEach { definitionFile ->
90+
val target = definitionFile.nameWithoutExtension.substringBeforeLast('_')
91+
92+
val deployDir = getDeployDir(workingDir, target)
93+
deployDirs += deployDir
94+
95+
val spdxFile = deployDir.findSpdxFiles().find { it.name == "recipe-$target.spdx.json" }
96+
if (spdxFile != null) {
97+
logger.info { "Not creating SPDX files for target '$target' as it already exists at '$spdxFile'." }
98+
} else {
99+
logger.info { "Creating SPDX files for target '$target'..." }
100+
101+
// This implicitly triggers the build and can take a very long time.
102+
val duration = measureTime { createSpdx(workingDir, target) }
103+
104+
logger.info { "Creating SPDX files for target '$target' took $duration." }
105+
}
106+
}
107+
108+
if (!scriptFile.delete()) logger.warn { "Unable to delete the temporary '$scriptFile' file." }
109+
if (!spdxConfFile.delete()) logger.warn { "Unable to delete the temporary '$spdxConfFile' file." }
110+
111+
val commonDeployDir = deployDirs.singleOrNull() ?: getCommonParentFile(deployDirs)
112+
val spdxFiles = commonDeployDir.findSpdxFiles().toList()
113+
114+
logger.info { "Found ${spdxFiles.size} SPDX file(s) in '$commonDeployDir'." }
115+
116+
return spdxManager.resolveDependencies(spdxFiles, labels)
117+
}
118+
119+
override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> =
120+
throw NotImplementedError("This function is not supported for $managerName.")
121+
122+
private fun getDeployDir(workingDir: File, target: String): File {
123+
val bitbakeEnv = runBitBake(workingDir, "-e", target)
124+
return bitbakeEnv.stdout.lineSequence().mapNotNull { it.withoutPrefix("DEPLOY_DIR=") }.first()
125+
.let { File(it.removeSurrounding("\"")) }
126+
}
127+
128+
private fun createSpdx(workingDir: File, target: String) =
129+
runBitBake(workingDir, "-r", spdxConfFile.absolutePath, "-c", "create_spdx", target)
130+
131+
private fun File.findSpdxFiles() = resolve("spdx").walk().filter { it.isFile && it.name.endsWith(".spdx.json") }
132+
133+
private fun runBitBake(workingDir: File, vararg args: String): ProcessCapture =
134+
ProcessCapture(scriptFile.absolutePath, workingDir.absolutePath, *args, workingDir = workingDir)
135+
.requireSuccess()
136+
137+
internal fun getBitBakeVersion(workingDir: File): String =
138+
runBitBake(workingDir, "--version").stdout.lineSequence().first {
139+
it.startsWith("BitBake Build Tool")
140+
}.substringAfterLast(' ')
141+
142+
private fun extractResourceToTempFile(resourceName: String): File {
143+
val prefix = resourceName.substringBefore('.')
144+
val suffix = resourceName.substringAfter(prefix)
145+
val scriptFile = createOrtTempFile(prefix, suffix)
146+
val script = checkNotNull(javaClass.getResource("/$resourceName")).readText()
147+
148+
return scriptFile.apply { writeText(script) }
149+
}
150+
}
151+
152+
private const val INIT_SCRIPT_NAME = "oe-init-build-env"
153+
private const val BITBAKE_SCRIPT_NAME = "bitbake.sh"
154+
private const val SPDX_CONF_NAME = "spdx.conf"
155+
156+
private fun File.searchUpwardsForFile(searchFileName: String): File? {
157+
if (!isDirectory) return null
158+
159+
var currentDir: File? = absoluteFile
160+
161+
while (currentDir != null && !currentDir.resolve(searchFileName).isFile) {
162+
currentDir = currentDir.parentFile
163+
}
164+
165+
return currentDir
166+
}

‎plugins/package-managers/bitbake/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageManagerFactory

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
org.ossreviewtoolkit.plugins.packagemanagers.bitbake.BitBake$Factory

‎plugins/package-managers/bitbake/src/main/resources/bitbake.sh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
# SPDX-License-Identifier: Apache-2.0
18+
# License-Filename: LICENSE
19+
20+
BUILD_DIR=$1; shift
21+
22+
if [ -z "$BBPATH" ] && [ -f oe-init-build-env ]; then
23+
# Initialize the build environment.
24+
. oe-init-build-env "$BUILD_DIR"
25+
fi
26+
27+
bitbake "$@"

0 commit comments

Comments
 (0)