diff --git a/Gopkg.lock b/Gopkg.lock index 67a74e8..7432212 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -64,6 +64,19 @@ ] revision = "30a6720f2ee35b4ba0f3660275e0b98600c35f62" +[[projects]] + branch = "master" + name = "github.com/dsnet/compress" + packages = [ + ".", + "bzip2", + "bzip2/internal/sais", + "internal", + "internal/errors", + "internal/prefix" + ] + revision = "cc9eb1d7ad760af14e8f918698f745e80377af4f" + [[projects]] name = "github.com/elazarl/go-bindata-assetfs" packages = ["."] @@ -330,6 +343,12 @@ ] revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8" +[[projects]] + name = "github.com/mholt/archiver" + packages = ["."] + revision = "cdc68dd1f170b8dfc1a0d2231b5bb0967ed67006" + version = "v2.0" + [[projects]] branch = "master" name = "github.com/mitchellh/copystructure" @@ -360,6 +379,12 @@ packages = ["."] revision = "63d60e9d0dbc60cf9164e6510889b0db6683d98c" +[[projects]] + branch = "master" + name = "github.com/nwaples/rardecode" + packages = ["."] + revision = "e06696f847aeda6f39a8f0b7cdff193b7690aef6" + [[projects]] name = "github.com/oklog/run" packages = ["."] @@ -396,6 +421,17 @@ revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" +[[projects]] + name = "github.com/ulikunitz/xz" + packages = [ + ".", + "internal/hash", + "internal/xlog", + "lzma" + ] + revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" + version = "v0.5.4" + [[projects]] branch = "master" name = "golang.org/x/crypto" @@ -554,6 +590,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1bfe6899fe528f7eca8d4d39a5f40b82b556b85827e85dafde8cd572cf0e4bab" + inputs-digest = "c9cc9ee0fcb4fd3667adcfd902be1306ad79d963ce365c57d26165e4e8e29953" solver-name = "gps-cdcl" solver-version = 1 diff --git a/releaser/.gitignore b/releaser/.gitignore new file mode 100644 index 0000000..f95d069 --- /dev/null +++ b/releaser/.gitignore @@ -0,0 +1,2 @@ +releaser +vendor diff --git a/releaser/INSTALL_RECEIPT.json.tmpl b/releaser/INSTALL_RECEIPT.json.tmpl new file mode 100644 index 0000000..34d6b6e --- /dev/null +++ b/releaser/INSTALL_RECEIPT.json.tmpl @@ -0,0 +1,35 @@ +{ + "homebrew_version": "1.5.2", + "used_options": [], + "unused_options": [], + "built_as_bottle": true, + "poured_from_bottle": false, + "installed_as_dependency": false, + "installed_on_request": true, + "changed_files": [ + "INSTALL_RECEIPT.json" + ], + "time": null, + "source_modified_time": 1515536363, + "HEAD": null, + "stdlib": null, + "compiler": "clang", + "aliases": [], + "runtime_dependencies": [ + { + "full_name": "kubernetes-cli", + "version": "1.9.2" + } + ], + "source": { + "path": "@@HOMEBREW_PREFIX@@/Homebrew/Library/Taps/dollarshaveclub/homebrew-public/Formula/psst.rb", + "tap": "dollarshaveclub/public", + "spec": "stable", + "versions": { + "stable": "{{ .Release }}", + "devel": null, + "head": "HEAD", + "version_scheme": 0 + } + } +} diff --git a/releaser/main.go b/releaser/main.go new file mode 100644 index 0000000..80376d5 --- /dev/null +++ b/releaser/main.go @@ -0,0 +1,473 @@ +package main + +/* +Minimal tool to automate release creation. + +Create: +- git tag +- homebrew bottle +- linux tarball +- GitHub release with asset link(s) + +Update: +- Homebrew formula tap with new release & SHAs + +*/ + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + + "github.com/google/go-github/github" + "github.com/mholt/archiver" + "github.com/pkg/errors" + "github.com/spf13/pflag" + "golang.org/x/oauth2" +) + +const ( + repoOwner = "dollarshaveclub" + repoName = "psst" +) + +var rname, npath, commitsha, ghtoken, taprepo, tapref, fpath, ftpath, targetoslist string +var draft, prerelease, dobuild bool +var trowner, trname string +var hbrev, brbd uint +var osvs []string + +var logger = log.New(os.Stderr, "", log.LstdFlags) + +func ferr(msg string, args ...interface{}) { + fmt.Printf(msg+"\n", args...) + os.Exit(1) +} + +var ghc *github.Client + +func init() { + pflag.StringVar(&rname, "release", "", "release name (ex: v1.0.0)") + pflag.StringVar(&npath, "notes-path", "relnotes.md", "path to release notes") + pflag.StringVar(&commitsha, "commit", "", "commit SHA to release") + pflag.StringVar(&taprepo, "tap-repo", "dollarshaveclub/homebrew-public", "name of tap GitHub repository ([owner]/[repo])") + pflag.StringVar(&tapref, "tap-repo-ref", "master", "tap repository ref (branch/tag/SHA)") + pflag.StringVar(&fpath, "formula", "Formula/psst.rb", "path to formula within tap repo") + pflag.StringVar(&ftpath, "formula-template", "Formula/psst.rb.tmpl", "path to formula template within tap repo") + pflag.StringVar(&targetoslist, "macos-versions", "el_capitan,high_sierra,sierra", "Supported MacOS versions (comma-delimited)") + pflag.UintVar(&hbrev, "homebrew-rev", 1, "Homebrew revision (bump to force reinstall/rebuild)") + pflag.UintVar(&brbd, "bottle-rebuild", 1, "Bottle rebuild (bump to force bottle reinstall)") + pflag.BoolVar(&draft, "draft", false, "Draft release (unpublished)") + pflag.BoolVar(&prerelease, "prerelease", false, "Prerelease") + pflag.BoolVar(&dobuild, "build", true, "Build binaries first") + pflag.Parse() + trs := strings.Split(taprepo, "/") + if len(trs) != 2 { + ferr("malformed tap repo (expected [owner]/[repo]): %v", taprepo) + } + if rname == "" { + ferr("release name is required") + } + trowner = trs[0] + trname = trs[1] + osvs = strings.Split(targetoslist, ",") + if len(osvs) == 0 { + ferr("At least one MacOS version is required") + } + ghtoken = os.Getenv("GITHUB_TOKEN") + if ghtoken == "" { + ferr("GITHUB_TOKEN missing from environment") + } + if err := checkFiles(npath); err != nil { + ferr("file path error: %v", err) + } + checkLocalRepoVersion() + ghc = newGHClient() +} + +func newGHClient() *github.Client { + tc := oauth2.NewClient(context.Background(), oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: ghtoken}, + )) + return github.NewClient(tc) +} + +func checkLocalRepoVersion() { + cmd := exec.Command("git", "rev-parse", "HEAD") + out, err := cmd.Output() + if err != nil { + ferr("error getting git command output: %v", err) + } + if strings.TrimRight(string(out), "\n") != commitsha { + ferr("current git revision does not match requested release version: %v (expected %v)", string(out), commitsha) + } +} + +func checkFiles(paths ...string) error { + for _, p := range paths { + if _, err := os.Stat(p); err != nil { + return errors.Wrap(err, "file error") + } + } + return nil +} + +func createGitTag() error { + msg := fmt.Sprintf("release %v", rname) + ot := "commit" + tag := github.Tag{ + Tag: &rname, + Message: &msg, + Object: &github.GitObject{ + Type: &ot, + SHA: &commitsha, + }, + } + log.Printf("creating tag...\n") + _, _, err := ghc.Git.CreateTag(context.Background(), repoOwner, repoName, &tag) + if err != nil { + return errors.Wrap(err, "error creating tag") + } + refstr := fmt.Sprintf("refs/tags/%v", rname) + objt := "commit" + ref := github.Reference{ + Ref: &refstr, + Object: &github.GitObject{ + Type: &objt, + SHA: &commitsha, + }, + } + log.Printf("creating tag ref...\n") + _, _, err = ghc.Git.CreateRef(context.Background(), repoOwner, repoName, &ref) + if err != nil { + return errors.Wrap(err, "error creating tag ref") + } + return nil +} + +type bottleDefinition struct { + Hash string + TargetOS string +} + +type formulaTemplateData struct { + Tag string + CommitSHA string + HomebrewRevision uint + BaseDownloadURL string + Bottled bool + BottleRebuild uint + BottleDefs []bottleDefinition +} + +func (ftd *formulaTemplateData) populate(bdefs []bottleDefinition) { + ftd.Tag = rname + ftd.CommitSHA = commitsha + ftd.HomebrewRevision = hbrev + ftd.BaseDownloadURL = fmt.Sprintf("https://github.com/%v/%v/releases/download/%v/", repoOwner, repoName, rname) + ftd.BottleRebuild = brbd + ftd.Bottled = true + ftd.BottleDefs = bdefs +} + +const header = "# GENERATED FROM TEMPLATE. DO NOT EDIT!\n" + +// generateFormula fetches the template from github, executes the template with ftd and returns the raw data or error, if any +func generateFormula(ftd formulaTemplateData) ([]byte, error) { + logger.Printf("Generating Homebrew formula") + // get template + fc, _, _, err := ghc.Repositories.GetContents(context.Background(), trowner, trname, ftpath, &github.RepositoryContentGetOptions{Ref: tapref}) + if err != nil { + return nil, errors.Wrap(err, "error getting formula template") + } + rt, err := fc.GetContent() + if err != nil { + return nil, errors.Wrap(err, "error getting formula template content") + } + // generate new formula + tmpl, err := template.New("formula").Parse(rt) + if err != nil { + return nil, errors.Wrap(err, "error parsing formula template") + } + buf := bytes.NewBuffer([]byte{}) + if err = tmpl.Execute(buf, &ftd); err != nil { + return nil, errors.Wrap(err, "error executing template") + } + return append([]byte(header), buf.Bytes()...), nil +} + +func pushFormula(fd []byte) error { + logger.Printf("Pushing Homebrew formula") + // Get the current file for the SHA + fc, _, _, err := ghc.Repositories.GetContents(context.Background(), trowner, trname, fpath, &github.RepositoryContentGetOptions{Ref: tapref}) + if err != nil { + return errors.Wrap(err, "error getting formula contents") + } + sp := func(s string) *string { + return &s + } + _, _, err = ghc.Repositories.UpdateFile(context.Background(), trowner, trname, fpath, &github.RepositoryContentFileOptions{ + Message: sp(fmt.Sprintf("updated for release %v", rname)), + Content: fd, + SHA: fc.SHA, + Branch: &tapref, + }) + if err != nil { + return errors.Wrap(err, "error updating formula") + } + return nil +} + +const ( + linuxBinName = "psst-linux-amd64" +) + +var buildopts = []string{"-ldflags", "-X github.com/dollarshaveclub/psst/internal/version.CommitSHA=%v -X github.com/dollarshaveclub/psst/internal/version.Version=%v -X $(REPO)/cmd.CompiledDirectory=github -X $(REPO)/cmd.CompiledStorage=vault -X $(REPO)/cmd.Org=dollarshaveclub"} + +func buildBins() error { + if err := os.MkdirAll("bins", os.ModeDir|0755); err != nil { + return errors.Wrap(err, "error creating bins directory") + } + cwd, err := os.Getwd() + if err != nil { + return errors.Wrap(err, "error getting working directory") + } + wd := filepath.Join(cwd, "..") + buildopts[1] = fmt.Sprintf(buildopts[1], commitsha, rname) + build := func(osn string) ([]byte, error) { + cmd := exec.Command("go", append([]string{"build"}, buildopts...)...) + cmd.Env = append(os.Environ(), []string{fmt.Sprintf("GOOS=%v", osn), "GOARCH=amd64"}...) + cmd.Dir = wd + return cmd.CombinedOutput() + } + logger.Printf("Building binaries...\n") + logger.Printf("...macOS amd64") + if out, err := build("darwin"); err != nil { + return errors.Wrapf(err, "error running build command: %s", out) + } + if err := os.Rename(filepath.Join(wd, "psst"), filepath.Join(cwd, "bins", "psst-darwin")); err != nil { + return errors.Wrap(err, "error renaming binary") + } + logger.Printf("...Linux amd64") + if out, err := build("linux"); err != nil { + return errors.Wrapf(err, "error running build command: %s", out) + } + lfn := filepath.Join(cwd, "bins", linuxBinName) + if err := os.Rename(filepath.Join(wd, "psst"), lfn); err != nil { + return errors.Wrap(err, "error renaming binary") + } + // compress linux binary + logger.Printf("...compressing Linux binary\n") + d, err := ioutil.ReadFile(lfn) + if err != nil { + return errors.Wrap(err, "error reading linux binary") + } + f, err := os.Create(lfn + ".gz") + if err != nil { + return errors.Wrap(err, "error creating compressed linux binary") + } + defer f.Close() + gw := gzip.NewWriter(f) + defer gw.Close() + if _, err := gw.Write(d); err != nil { + return errors.Wrap(err, "error writing compressed linux binary") + } + return nil +} + +// "copy" (link) a file if it doesn't exist +func cpifneeded(src, dest string) error { + if _, err := os.Stat(dest); err != nil { + if os.IsNotExist(err) { + return os.Link(src, dest) + } + return errors.Wrap(err, "error getting destination") + } + return nil +} + +var bottleNameTmpl = template.Must(template.New("bn").Parse("psst-{{ .Release }}_{{ .HomebrewRevision }}.{{ .OS }}.bottle.{{ .BottleRebuild }}.tar.gz")) + +// createBottle synthetically creates a bottle tarball returning the bottle definitions, local bottle filenames and error if any +func createBottle() ([]bottleDefinition, []string, error) { + logger.Printf("Creating Homebrew bottle...\n") + cwd, err := os.Getwd() + if err != nil { + return nil, nil, errors.Wrap(err, "error getting working directory") + } + basepath := filepath.Join(".", "psst", rname) + binpath := filepath.Join(basepath, "bin") + if err := os.MkdirAll(binpath, os.ModeDir|0755); err != nil { + return nil, nil, errors.Wrap(err, "error creating bottle directory path") + } + // .brew + if err := os.MkdirAll(filepath.Join(basepath, ".brew"), os.ModeDir|0755); err != nil { + return nil, nil, errors.Wrap(err, "error creating .brew directory") + } + // copy README + if err := cpifneeded(filepath.Join(cwd, "..", "README.md"), filepath.Join(basepath, "README.md")); err != nil { + return nil, nil, errors.Wrap(err, "error copying README") + } + // copy binary + if err := cpifneeded(filepath.Join("bins", "psst-darwin"), filepath.Join(binpath, "psst")); err != nil { + return nil, nil, errors.Wrap(err, "error copying binary") + } + // INSTALL_RECEIPT.json + ir, err := ioutil.ReadFile("INSTALL_RECEIPT.json.tmpl") + if err != nil { + return nil, nil, errors.Wrap(err, "error reading install receipt template") + } + tmpl, err := template.New("instrcpt").Parse(string(ir)) + d := struct { + Release string + OS string + HomebrewRevision uint + BottleRebuild uint + }{ + Release: rname, + HomebrewRevision: hbrev, + BottleRebuild: brbd, + } + buf := bytes.NewBuffer([]byte{}) + if err := tmpl.Execute(buf, &d); err != nil { + return nil, nil, errors.Wrap(err, "error executing install receipt template") + } + if err := ioutil.WriteFile(filepath.Join(basepath, "INSTALL_RECEIPT.json"), buf.Bytes(), os.ModePerm); err != nil { + return nil, nil, errors.Wrap(err, "error writing install receipt") + } + // tar it up + if err := os.MkdirAll("bottle", os.ModeDir|0755); err != nil { + return nil, nil, errors.Wrap(err, "error creating bottle directory") + } + buf = bytes.NewBuffer([]byte{}) + d.OS = osvs[0] + if err := bottleNameTmpl.Execute(buf, &d); err != nil { + return nil, nil, errors.Wrap(err, "error executing bottle filename template: "+d.OS) + } + bp := filepath.Join("bottle", buf.String()) + if err := archiver.TarGz.Make(bp, []string{"psst"}); err != nil { + return nil, nil, errors.Wrap(err, "error creating bottle tarball") + } + // Get hash of bottle, populate bottle definitions + bd, err := ioutil.ReadFile(bp) + if err != nil { + return nil, nil, errors.Wrap(err, "error reading bottle") + } + sha := fmt.Sprintf("%x", sha256.Sum256(bd)) + bdefs := []bottleDefinition{ + bottleDefinition{ + Hash: sha, + TargetOS: osvs[0], + }, + } + lps := []string{bp} + // link other bottles + for _, osn := range osvs[1:] { + d.OS = osn + buf = bytes.NewBuffer([]byte{}) + if err := bottleNameTmpl.Execute(buf, &d); err != nil { + return nil, nil, errors.Wrap(err, "error executing bottle filename template: "+d.OS) + } + p := filepath.Join("bottle", buf.String()) + if err := cpifneeded(bp, p); err != nil { + return nil, nil, errors.Wrap(err, "error linking bottle") + } + lps = append(lps, p) + bdefs = append(bdefs, bottleDefinition{ + Hash: sha, + TargetOS: osn, + }) + } + + return bdefs, lps, nil +} + +func createGHRelease(assetpaths []string) error { + rel := github.RepositoryRelease{ + TagName: &rname, + //TargetCommitish: &commitsha, + Name: &rname, + Draft: &draft, + Prerelease: &prerelease, + } + nd, err := ioutil.ReadFile(npath) + if err != nil { + return errors.Wrap(err, "error reading release notes") + } + notes := string(nd) + rel.Body = ¬es + logger.Printf("Creating GitHub release") + ro, _, err := ghc.Repositories.CreateRelease(context.Background(), repoOwner, repoName, &rel) + if err != nil { + return errors.Wrap(err, "error creating release") + } + for _, ap := range assetpaths { + f, err := os.Open(ap) + if err != nil { + return errors.Wrap(err, "error opening asset") + } + defer f.Close() + logger.Printf("Uploading asset %v...", ap) + resp, _, err := ghc.Repositories.UploadReleaseAsset(context.Background(), repoOwner, repoName, *ro.ID, &github.UploadOptions{Name: filepath.Base(ap)}, f) + if err != nil { + return errors.Wrap(err, "error uploading asset") + } + logger.Printf("...%v\n", resp.GetBrowserDownloadURL()) + } + return nil +} + +func cleanup() error { + logger.Printf("Cleaning up") + for _, p := range []string{"./bins", "./bottle", "./psst"} { + if err := os.RemoveAll(p); err != nil { + return errors.Wrap(err, "error removing path") + } + } + return nil +} + +func main() { + if dobuild { + if err := buildBins(); err != nil { + ferr("error building binaries: %v", err) + } + } + bds, lps, err := createBottle() + if err != nil { + ferr("error creating bottle: %v", err) + } + ftd := formulaTemplateData{} + ftd.populate(bds) + fd, err := generateFormula(ftd) + if err != nil { + ferr("error generating formula: %v", err) + } + if err = pushFormula(fd); err != nil { + ferr("error pushing formula: %v", err) + } + if err := createGitTag(); err != nil { + ferr("error creating tag: %v", err) + } + cwd, err := os.Getwd() + if err != nil { + ferr("error getting working directory: %v", err) + } + assetpaths := append([]string{filepath.Join(cwd, "bins", linuxBinName+".gz")}, lps...) + if err = createGHRelease(assetpaths); err != nil { + ferr("error creating GitHub release: %v", err) + } + if err := cleanup(); err != nil { + ferr("error cleaning up: %v", err) + } + logger.Printf("Done") +} diff --git a/releaser/relnotes.md b/releaser/relnotes.md new file mode 100644 index 0000000..17d43e9 --- /dev/null +++ b/releaser/relnotes.md @@ -0,0 +1,3 @@ +# Initial Release + +- Adds command for sharing secrets using GitHub as a user directory and Vault as a secure storage backend. \ No newline at end of file diff --git a/releaser/relnotes.md.tmpl b/releaser/relnotes.md.tmpl new file mode 100644 index 0000000..e87f1aa --- /dev/null +++ b/releaser/relnotes.md.tmpl @@ -0,0 +1,26 @@ +# psst v?.?.? + +## Installation + +From an existing version: +``` +brew upgrade psst +``` + +From scratch: +``` +brew tap dollarshaveclub/public +brew install psst +``` + +## Features + +- include new features here + +## Improvements + +- include bug fixes and improvements here + +## Known Issues + +- include known issues here (see previous release and copy existing known issues) \ No newline at end of file diff --git a/vendor/github.com/dsnet/compress/.travis.yml b/vendor/github.com/dsnet/compress/.travis.yml new file mode 100644 index 0000000..9fb6745 --- /dev/null +++ b/vendor/github.com/dsnet/compress/.travis.yml @@ -0,0 +1,24 @@ +sudo: false +language: go +before_install: + - curl -L https://github.com/google/brotli/archive/v1.0.2.tar.gz | tar -zxv + - (cd brotli-1.0.2 && mkdir out && cd out && ../configure-cmake && make && sudo make install) + - rm -rf brotli-1.0.2 + - curl -L https://github.com/facebook/zstd/archive/v1.3.2.tar.gz | tar -zxv + - (cd zstd-1.3.2 && sudo make install) + - rm -rf zstd-1.3.2 + - sudo ldconfig +before_script: + - go get github.com/golang/lint/golint + - go get honnef.co/go/tools/cmd/staticcheck +matrix: + include: + - go: 1.7.x + script: go test -v -race ./... + - go: 1.x + script: ./ztest.sh + - go: master + script: go test -v -race ./... + allow_failures: + - go: master + fast_finish: true diff --git a/vendor/github.com/dsnet/compress/LICENSE.md b/vendor/github.com/dsnet/compress/LICENSE.md new file mode 100644 index 0000000..945b396 --- /dev/null +++ b/vendor/github.com/dsnet/compress/LICENSE.md @@ -0,0 +1,24 @@ +Copyright © 2015, Joe Tsai and The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. +* Neither the copyright holder nor the names of its contributors may be used to +endorse or promote products derived from this software without specific prior +written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/dsnet/compress/README.md b/vendor/github.com/dsnet/compress/README.md new file mode 100644 index 0000000..f256944 --- /dev/null +++ b/vendor/github.com/dsnet/compress/README.md @@ -0,0 +1,75 @@ +# Collection of compression libraries for Go # + +[![GoDoc](https://godoc.org/github.com/dsnet/compress/cmp?status.svg)](https://godoc.org/github.com/dsnet/compress) +[![Build Status](https://travis-ci.org/dsnet/compress.svg?branch=master)](https://travis-ci.org/dsnet/compress) +[![Report Card](https://goreportcard.com/badge/github.com/dsnet/compress)](https://goreportcard.com/report/github.com/dsnet/compress) + +## Introduction ## + +**NOTE: This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason.** + +This repository hosts a collection of compression related libraries. The goal of this project is to provide pure Go implementations for popular compression algorithms beyond what the Go standard library provides. The goals for these packages are as follows: +* Maintainable: That the code remains well documented, well tested, readable, easy to maintain, and easy to verify that it conforms to the specification for the format being implemented. +* Performant: To be able to compress and decompress within at least 80% of the rates that the C implementations are able to achieve. +* Flexible: That the code provides low-level and fine granularity control over the compression streams similar to what the C APIs would provide. + +Of these three, the first objective is often at odds with the other two objectives and provides interesting challenges. Higher performance can often be achieved by muddling abstraction layers or using non-intuitive low-level primitives. Also, more features and functionality, while useful in some situations, often complicates the API. Thus, this package will attempt to satisfy all the goals, but will defer to favoring maintainability when the performance or flexibility benefits are not significant enough. + + +## Library Status ## + +For the packages available, only some features are currently implemented: + +| Package | Reader | Writer | +| ------- | :----: | :----: | +| brotli | :white_check_mark: | | +| bzip2 | :white_check_mark: | :white_check_mark: | +| flate | :white_check_mark: | | +| xflate | :white_check_mark: | :white_check_mark: | + +This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason. When the library becomes more mature, it is planned to eventually conform to some strict versioning scheme like [Semantic Versioning](http://semver.org/). + +However, in the meanwhile, this library does provide some basic API guarantees. For the types defined below, the method signatures are guaranteed to not change. Note that the author still reserves the right to change the fields within each ```Reader``` and ```Writer``` structs. +```go +type ReaderConfig struct { ... } +type Reader struct { ... } + func NewReader(io.Reader, *ReaderConfig) (*Reader, error) { ... } + func (*Reader) Read([]byte) (int, error) { ... } + func (*Reader) Close() error { ... } + +type WriterConfig struct { ... } +type Writer struct { ... } + func NewWriter(io.Writer, *WriterConfig) (*Writer, error) { ... } + func (*Writer) Write([]byte) (int, error) { ... } + func (*Writer) Close() error { ... } +``` + +To see what work still remains, see the [Task List](https://github.com/dsnet/compress/wiki/Task-List). + +## Performance ## + +See [Performance Metrics](https://github.com/dsnet/compress/wiki/Performance-Metrics). + + +## Frequently Asked Questions ## + +See [Frequently Asked Questions](https://github.com/dsnet/compress/wiki/Frequently-Asked-Questions). + + +## Installation ## + +Run the command: + +```go get -u github.com/dsnet/compress``` + +This library requires `Go1.7` or higher in order to build. + + +## Packages ## + +| Package | Description | +| :------ | :---------- | +| [brotli](http://godoc.org/github.com/dsnet/compress/brotli) | Package brotli implements the Brotli format, described in RFC 7932. | +| [bzip2](http://godoc.org/github.com/dsnet/compress/bzip2) | Package bzip2 implements the BZip2 compressed data format. | +| [flate](http://godoc.org/github.com/dsnet/compress/flate) | Package flate implements the DEFLATE format, described in RFC 1951. | +| [xflate](http://godoc.org/github.com/dsnet/compress/xflate) | Package xflate implements the XFLATE format, an random-access extension to DEFLATE. | diff --git a/vendor/github.com/dsnet/compress/api.go b/vendor/github.com/dsnet/compress/api.go new file mode 100644 index 0000000..f80a923 --- /dev/null +++ b/vendor/github.com/dsnet/compress/api.go @@ -0,0 +1,74 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package compress is a collection of compression libraries. +package compress + +import ( + "bufio" + "io" + + "github.com/dsnet/compress/internal/errors" +) + +// The Error interface identifies all compression related errors. +type Error interface { + error + CompressError() + + // IsDeprecated reports the use of a deprecated and unsupported feature. + IsDeprecated() bool + + // IsCorrupted reports whether the input stream was corrupted. + IsCorrupted() bool +} + +var _ Error = errors.Error{} + +// ByteReader is an interface accepted by all decompression Readers. +// It guarantees that the decompressor never reads more data than is necessary +// from the underlying io.Reader. +type ByteReader interface { + io.Reader + io.ByteReader +} + +var _ ByteReader = (*bufio.Reader)(nil) + +// BufferedReader is an interface accepted by all decompression Readers. +// It guarantees that the decompressor never reads more data than is necessary +// from the underlying io.Reader. Since BufferedReader allows a decompressor +// to peek at bytes further along in the stream without advancing the read +// pointer, decompression can experience a significant performance gain when +// provided a reader that satisfies this interface. Thus, a decompressor will +// prefer this interface over ByteReader for performance reasons. +// +// The bufio.Reader satisfies this interface. +type BufferedReader interface { + io.Reader + + // Buffered returns the number of bytes currently buffered. + // + // This value becomes invalid following the next Read/Discard operation. + Buffered() int + + // Peek returns the next n bytes without advancing the reader. + // + // If Peek returns fewer than n bytes, it also returns an error explaining + // why the peek is short. Peek must support peeking of at least 8 bytes. + // If 0 <= n <= Buffered(), Peek is guaranteed to succeed without reading + // from the underlying io.Reader. + // + // This result becomes invalid following the next Read/Discard operation. + Peek(n int) ([]byte, error) + + // Discard skips the next n bytes, returning the number of bytes discarded. + // + // If Discard skips fewer than n bytes, it also returns an error. + // If 0 <= n <= Buffered(), Discard is guaranteed to succeed without reading + // from the underlying io.Reader. + Discard(n int) (int, error) +} + +var _ BufferedReader = (*bufio.Reader)(nil) diff --git a/vendor/github.com/dsnet/compress/bzip2/bwt.go b/vendor/github.com/dsnet/compress/bzip2/bwt.go new file mode 100644 index 0000000..44a2541 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/bwt.go @@ -0,0 +1,110 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import "github.com/dsnet/compress/bzip2/internal/sais" + +// The Burrows-Wheeler Transform implementation used here is based on the +// Suffix Array by Induced Sorting (SA-IS) methodology by Nong, Zhang, and Chan. +// This implementation uses the sais algorithm originally written by Yuta Mori. +// +// The SA-IS algorithm runs in O(n) and outputs a Suffix Array. There is a +// mathematical relationship between Suffix Arrays and the Burrows-Wheeler +// Transform, such that a SA can be converted to a BWT in O(n) time. +// +// References: +// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf +// https://github.com/cscott/compressjs/blob/master/lib/BWT.js +// https://www.quora.com/How-can-I-optimize-burrows-wheeler-transform-and-inverse-transform-to-work-in-O-n-time-O-n-space +type burrowsWheelerTransform struct { + buf []byte + sa []int + perm []uint32 +} + +func (bwt *burrowsWheelerTransform) Encode(buf []byte) (ptr int) { + if len(buf) == 0 { + return -1 + } + + // TODO(dsnet): Find a way to avoid the duplicate input string method. + // We only need to do this because suffix arrays (by definition) only + // operate non-wrapped suffixes of a string. On the other hand, + // the BWT specifically used in bzip2 operate on a strings that wrap-around + // when being sorted. + + // Step 1: Concatenate the input string to itself so that we can use the + // suffix array algorithm for bzip2's variant of BWT. + n := len(buf) + bwt.buf = append(append(bwt.buf[:0], buf...), buf...) + if cap(bwt.sa) < 2*n { + bwt.sa = make([]int, 2*n) + } + t := bwt.buf[:2*n] + sa := bwt.sa[:2*n] + + // Step 2: Compute the suffix array (SA). The input string, t, will not be + // modified, while the results will be written to the output, sa. + sais.ComputeSA(t, sa) + + // Step 3: Convert the SA to a BWT. Since ComputeSA does not mutate the + // input, we have two copies of the input; in buf and buf2. Thus, we write + // the transformation to buf, while using buf2. + var j int + buf2 := t[n:] + for _, i := range sa { + if i < n { + if i == 0 { + ptr = j + i = n + } + buf[j] = buf2[i-1] + j++ + } + } + return ptr +} + +func (bwt *burrowsWheelerTransform) Decode(buf []byte, ptr int) { + if len(buf) == 0 { + return + } + + // Step 1: Compute cumm, where cumm[ch] reports the total number of + // characters that precede the character ch in the alphabet. + var cumm [256]int + for _, v := range buf { + cumm[v]++ + } + var sum int + for i, v := range cumm { + cumm[i] = sum + sum += v + } + + // Step 2: Compute perm, where perm[ptr] contains a pointer to the next + // byte in buf and the next pointer in perm itself. + if cap(bwt.perm) < len(buf) { + bwt.perm = make([]uint32, len(buf)) + } + perm := bwt.perm[:len(buf)] + for i, b := range buf { + perm[cumm[b]] = uint32(i) + cumm[b]++ + } + + // Step 3: Follow each pointer in perm to the next byte, starting with the + // origin pointer. + if cap(bwt.buf) < len(buf) { + bwt.buf = make([]byte, len(buf)) + } + buf2 := bwt.buf[:len(buf)] + i := perm[ptr] + for j := range buf2 { + buf2[j] = buf[i] + i = perm[i] + } + copy(buf, buf2) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/common.go b/vendor/github.com/dsnet/compress/bzip2/common.go new file mode 100644 index 0000000..c633981 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/common.go @@ -0,0 +1,110 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package bzip2 implements the BZip2 compressed data format. +// +// Canonical C implementation: +// http://bzip.org +// +// Unofficial format specification: +// https://github.com/dsnet/compress/blob/master/doc/bzip2-format.pdf +package bzip2 + +import ( + "fmt" + "hash/crc32" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" +) + +// There does not exist a formal specification of the BZip2 format. As such, +// much of this work is derived by either reverse engineering the original C +// source code or using secondary sources. +// +// Significant amounts of fuzz testing is done to ensure that outputs from +// this package is properly decoded by the C library. Furthermore, we test that +// both this package and the C library agree about what inputs are invalid. +// +// Compression stack: +// Run-length encoding 1 (RLE1) +// Burrows-Wheeler transform (BWT) +// Move-to-front transform (MTF) +// Run-length encoding 2 (RLE2) +// Prefix encoding (PE) +// +// References: +// http://bzip.org/ +// https://en.wikipedia.org/wiki/Bzip2 +// https://code.google.com/p/jbzip2/ + +const ( + BestSpeed = 1 + BestCompression = 9 + DefaultCompression = 6 +) + +const ( + hdrMagic = 0x425a // Hex of "BZ" + blkMagic = 0x314159265359 // BCD of PI + endMagic = 0x177245385090 // BCD of sqrt(PI) + + blockSize = 100000 +) + +func errorf(c int, f string, a ...interface{}) error { + return errors.Error{Code: c, Pkg: "bzip2", Msg: fmt.Sprintf(f, a...)} +} + +func panicf(c int, f string, a ...interface{}) { + errors.Panic(errorf(c, f, a...)) +} + +// errWrap converts a lower-level errors.Error to be one from this package. +// The replaceCode passed in will be used to replace the code for any errors +// with the errors.Invalid code. +// +// For the Reader, set this to errors.Corrupted. +// For the Writer, set this to errors.Internal. +func errWrap(err error, replaceCode int) error { + if cerr, ok := err.(errors.Error); ok { + if errors.IsInvalid(cerr) { + cerr.Code = replaceCode + } + err = errorf(cerr.Code, "%s", cerr.Msg) + } + return err +} + +var errClosed = errorf(errors.Closed, "") + +// crc computes the CRC-32 used by BZip2. +// +// The CRC-32 computation in bzip2 treats bytes as having bits in big-endian +// order. That is, the MSB is read before the LSB. Thus, we can use the +// standard library version of CRC-32 IEEE with some minor adjustments. +// +// The byte array is used as an intermediate buffer to swap the bits of every +// byte of the input. +type crc struct { + val uint32 + buf [256]byte +} + +// update computes the CRC-32 of appending buf to c. +func (c *crc) update(buf []byte) { + cval := internal.ReverseUint32(c.val) + for len(buf) > 0 { + n := len(buf) + if n > len(c.buf) { + n = len(c.buf) + } + for i, b := range buf[:n] { + c.buf[i] = internal.ReverseLUT[b] + } + cval = crc32.Update(cval, crc32.IEEETable, c.buf[:n]) + buf = buf[n:] + } + c.val = internal.ReverseUint32(cval) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go b/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go new file mode 100644 index 0000000..ddd32f5 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/fuzz_off.go @@ -0,0 +1,13 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build !gofuzz + +// This file exists to suppress fuzzing details from release builds. + +package bzip2 + +type fuzzReader struct{} + +func (*fuzzReader) updateChecksum(int64, uint32) {} diff --git a/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go b/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go new file mode 100644 index 0000000..5412235 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/fuzz_on.go @@ -0,0 +1,77 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build gofuzz + +// This file exists to export internal implementation details for fuzz testing. + +package bzip2 + +func ForwardBWT(buf []byte) (ptr int) { + var bwt burrowsWheelerTransform + return bwt.Encode(buf) +} + +func ReverseBWT(buf []byte, ptr int) { + var bwt burrowsWheelerTransform + bwt.Decode(buf, ptr) +} + +type fuzzReader struct { + Checksums Checksums +} + +// updateChecksum updates Checksums. +// +// If a valid pos is provided, it appends the (pos, val) pair to the slice. +// Otherwise, it will update the last record with the new value. +func (fr *fuzzReader) updateChecksum(pos int64, val uint32) { + if pos >= 0 { + fr.Checksums = append(fr.Checksums, Checksum{pos, val}) + } else { + fr.Checksums[len(fr.Checksums)-1].Value = val + } +} + +type Checksum struct { + Offset int64 // Bit offset of the checksum + Value uint32 // Checksum value +} + +type Checksums []Checksum + +// Apply overwrites all checksum fields in d with the ones in cs. +func (cs Checksums) Apply(d []byte) []byte { + d = append([]byte(nil), d...) + for _, c := range cs { + setU32(d, c.Offset, c.Value) + } + return d +} + +func setU32(d []byte, pos int64, val uint32) { + for i := uint(0); i < 32; i++ { + bpos := uint64(pos) + uint64(i) + d[bpos/8] &= ^byte(1 << (7 - bpos%8)) + d[bpos/8] |= byte(val>>(31-i)) << (7 - bpos%8) + } +} + +// Verify checks that all checksum fields in d matches those in cs. +func (cs Checksums) Verify(d []byte) bool { + for _, c := range cs { + if getU32(d, c.Offset) != c.Value { + return false + } + } + return true +} + +func getU32(d []byte, pos int64) (val uint32) { + for i := uint(0); i < 32; i++ { + bpos := uint64(pos) + uint64(i) + val |= (uint32(d[bpos/8] >> (7 - bpos%8))) << (31 - i) + } + return val +} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go new file mode 100644 index 0000000..cd4eee8 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/common.go @@ -0,0 +1,28 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package sais implements a linear time suffix array algorithm. +package sais + +//go:generate go run sais_gen.go byte sais_byte.go +//go:generate go run sais_gen.go int sais_int.go + +// This package ports the C sais implementation by Yuta Mori. The ports are +// located in sais_byte.go and sais_int.go, which are identical to each other +// except for the types. Since Go does not support generics, we use generators to +// create the two files. +// +// References: +// https://sites.google.com/site/yuta256/sais +// https://www.researchgate.net/publication/221313676_Linear_Time_Suffix_Array_Construction_Using_D-Critical_Substrings +// https://www.researchgate.net/publication/224176324_Two_Efficient_Algorithms_for_Linear_Time_Suffix_Array_Construction + +// ComputeSA computes the suffix array of t and places the result in sa. +// Both t and sa must be the same length. +func ComputeSA(t []byte, sa []int) { + if len(sa) != len(t) { + panic("mismatching sizes") + } + computeSA_byte(t, sa, 0, len(t), 256) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go new file mode 100644 index 0000000..01b8529 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_byte.go @@ -0,0 +1,661 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Code generated by sais_gen.go. DO NOT EDIT. + +// ==================================================== +// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// ==================================================== + +package sais + +func getCounts_byte(T []byte, C []int, n, k int) { + var i int + for i = 0; i < k; i++ { + C[i] = 0 + } + for i = 0; i < n; i++ { + C[T[i]]++ + } +} + +func getBuckets_byte(C, B []int, k int, end bool) { + var i, sum int + if end { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum + } + } else { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum - C[i] + } + } +} + +func sortLMS1_byte(T []byte, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + b-- + if int(T[j]) > c1 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS1_byte(T []byte, SA []int, n, m int) int { + var i, j, p, q, plen, qlen, name int + var c0, c1 int + var diff bool + + // Compact all the sorted substrings into the first m items of SA. + // 2*m must be not larger than n (provable). + for i = 0; SA[i] < 0; i++ { + SA[i] = ^SA[i] + } + if i < m { + for j, i = i, i+1; ; i++ { + if p = SA[i]; p < 0 { + SA[j] = ^p + j++ + SA[i] = 0 + if j == m { + break + } + } + } + } + + // Store the length of all substrings. + i = n - 1 + j = n - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + SA[m+((i+1)>>1)] = j - i + j = i + 1 + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + // Find the lexicographic names of all substrings. + name = 0 + qlen = 0 + for i, q = 0, n; i < m; i++ { + p = SA[i] + plen = SA[m+(p>>1)] + diff = true + if (plen == qlen) && ((q + plen) < n) { + for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { + } + if j == plen { + diff = false + } + } + if diff { + name++ + q = p + qlen = plen + } + SA[m+(p>>1)] = name + } + return name +} + +func sortLMS2_byte(T []byte, SA, C, B, D []int, n, k int) { + var b, i, j, t, d int + var c0, c1 int + + // Compute SAl. + getBuckets_byte(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + t = 1 + } else { + t = 0 + } + j += n + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i, d = 0, 0; i < n; i++ { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) < c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + for i = n - 1; 0 <= i; i-- { + if SA[i] > 0 { + if SA[i] < n { + SA[i] += n + for j = i - 1; SA[j] < n; j-- { + } + SA[j] -= n + i = j + } + } + } + + // Compute SAs. + getBuckets_byte(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i, d = n-1, d+1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) > c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + b-- + if t&1 > 0 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS2_byte(SA []int, n, m int) int { + var i, j, d, name int + + // Compact all the sorted LMS substrings into the first m items of SA. + name = 0 + for i = 0; SA[i] < 0; i++ { + j = ^SA[i] + if n <= j { + name += 1 + } + SA[i] = j + } + if i < m { + for d, i = i, i+1; ; i++ { + if j = SA[i]; j < 0 { + j = ^j + if n <= j { + name += 1 + } + SA[d] = j + d++ + SA[i] = 0 + if d == m { + break + } + } + } + } + if name < m { + // Store the lexicographic names. + for i, d = m-1, name+1; 0 <= i; i-- { + if j = SA[i]; n <= j { + j -= n + d-- + } + SA[m+(j>>1)] = d + } + } else { + // Unset flags. + for i = 0; i < m; i++ { + if j = SA[i]; n <= j { + j -= n + SA[i] = j + } + } + } + return name +} + +func induceSA_byte(T []byte, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + j = SA[i] + SA[i] = ^j + if j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_byte(T, C, n, k) + } + getBuckets_byte(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + b-- + if (j == 0) || (int(T[j-1]) > c1) { + SA[b] = ^j + } else { + SA[b] = j + } + } else { + SA[i] = ^j + } + } +} + +func computeSA_byte(T []byte, SA []int, fs, n, k int) { + const ( + minBucketSize = 512 + sortLMS2Limit = 0x3fffffff + ) + + var C, B, D, RA []int + var bo int // Offset of B relative to SA + var b, i, j, m, p, q, name, newfs int + var c0, c1 int + var flags uint + + if k <= minBucketSize { + C = make([]int, k) + if k <= fs { + bo = n + fs - k + B = SA[bo:] + flags = 1 + } else { + B = make([]int, k) + flags = 3 + } + } else if k <= fs { + C = SA[n+fs-k:] + if k <= fs-k { + bo = n + fs - 2*k + B = SA[bo:] + flags = 0 + } else if k <= 4*minBucketSize { + B = make([]int, k) + flags = 2 + } else { + B = C + flags = 8 + } + } else { + C = make([]int, k) + B = C + flags = 4 | 8 + } + if n <= sortLMS2Limit && 2 <= (n/k) { + if flags&1 > 0 { + if 2*k <= fs-k { + flags |= 32 + } else { + flags |= 16 + } + } else if flags == 0 && 2*k <= (fs-2*k) { + flags |= 32 + } + } + + // Stage 1: Reduce the problem by at least 1/2. + // Sort all the LMS-substrings. + getCounts_byte(T, C, n, k) + getBuckets_byte(C, B, k, true) // Find ends of buckets + for i = 0; i < n; i++ { + SA[i] = 0 + } + b = -1 + i = n - 1 + j = n + m = 0 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + if b >= 0 { + SA[b] = j + } + B[c1]-- + b = B[c1] + j = i + m++ + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + if m > 1 { + if flags&(16|32) > 0 { + if flags&16 > 0 { + D = make([]int, 2*k) + } else { + D = SA[bo-2*k:] + } + B[T[j+1]]++ + for i, j = 0, 0; i < k; i++ { + j += C[i] + if B[i] != j { + SA[B[i]] += n + } + D[i] = 0 + D[i+k] = 0 + } + sortLMS2_byte(T, SA, C, B, D, n, k) + name = postProcLMS2_byte(SA, n, m) + } else { + sortLMS1_byte(T, SA, C, B, n, k) + name = postProcLMS1_byte(T, SA, n, m) + } + } else if m == 1 { + SA[b] = j + 1 + name = 1 + } else { + name = 0 + } + + // Stage 2: Solve the reduced problem. + // Recurse if names are not yet unique. + if name < m { + newfs = n + fs - 2*m + if flags&(1|4|8) == 0 { + if k+name <= newfs { + newfs -= k + } else { + flags |= 8 + } + } + RA = SA[m+newfs:] + for i, j = m+(n>>1)-1, m-1; m <= i; i-- { + if SA[i] != 0 { + RA[j] = SA[i] - 1 + j-- + } + } + computeSA_int(RA, SA, newfs, m, name) + + i = n - 1 + j = m - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + RA[j] = i + 1 + j-- + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + for i = 0; i < m; i++ { + SA[i] = RA[SA[i]] + } + if flags&4 > 0 { + B = make([]int, k) + C = B + } + if flags&2 > 0 { + B = make([]int, k) + } + } + + // Stage 3: Induce the result for the original problem. + if flags&8 > 0 { + getCounts_byte(T, C, n, k) + } + // Put all left-most S characters into their buckets. + if m > 1 { + getBuckets_byte(C, B, k, true) // Find ends of buckets + i = m - 1 + j = n + p = SA[m-1] + c1 = int(T[p]) + for { + c0 = c1 + q = B[c0] + for q < j { + j-- + SA[j] = 0 + } + for { + j-- + SA[j] = p + if i--; i < 0 { + break + } + p = SA[i] + if c1 = int(T[p]); c1 != c0 { + break + } + } + if i < 0 { + break + } + } + for j > 0 { + j-- + SA[j] = 0 + } + } + induceSA_byte(T, SA, C, B, n, k) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go new file mode 100644 index 0000000..26bf628 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_gen.go @@ -0,0 +1,703 @@ +// Copyright 2017, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build ignore + +package main + +import ( + "bytes" + "go/format" + "io/ioutil" + "log" + "os" + "text/template" +) + +func main() { + if len(os.Args) != 3 { + log.Fatalf("Usage: %s GO_TYPE OUTPUT_FILE", os.Args[0]) + } + typ := os.Args[1] + path := os.Args[2] + + b := new(bytes.Buffer) + t := template.Must(template.New("source").Parse(source)) + if err := t.Execute(b, struct { + Type, GeneratedMessage string + }{typ, "// Code generated by sais_gen.go. DO NOT EDIT."}); err != nil { + log.Fatalf("Template.Execute error: %v", err) + } + out, err := format.Source(bytes.TrimSpace(b.Bytes())) + if err != nil { + log.Fatalf("format.Source error: %v", err) + } + if err := ioutil.WriteFile(path, out, 0644); err != nil { + log.Fatalf("ioutil.WriteFile error: %v", err) + } +} + +const source = ` +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +{{.GeneratedMessage}} + +// ==================================================== +// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// ==================================================== + +package sais + +func getCounts_{{.Type}}(T []{{.Type}}, C []int, n, k int) { + var i int + for i = 0; i < k; i++ { + C[i] = 0 + } + for i = 0; i < n; i++ { + C[T[i]]++ + } +} + +func getBuckets_{{.Type}}(C, B []int, k int, end bool) { + var i, sum int + if end { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum + } + } else { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum - C[i] + } + } +} + +func sortLMS1_{{.Type}}(T []{{.Type}}, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + b-- + if int(T[j]) > c1 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS1_{{.Type}}(T []{{.Type}}, SA []int, n, m int) int { + var i, j, p, q, plen, qlen, name int + var c0, c1 int + var diff bool + + // Compact all the sorted substrings into the first m items of SA. + // 2*m must be not larger than n (provable). + for i = 0; SA[i] < 0; i++ { + SA[i] = ^SA[i] + } + if i < m { + for j, i = i, i+1; ; i++ { + if p = SA[i]; p < 0 { + SA[j] = ^p + j++ + SA[i] = 0 + if j == m { + break + } + } + } + } + + // Store the length of all substrings. + i = n - 1 + j = n - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + SA[m+((i+1)>>1)] = j - i + j = i + 1 + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + // Find the lexicographic names of all substrings. + name = 0 + qlen = 0 + for i, q = 0, n; i < m; i++ { + p = SA[i] + plen = SA[m+(p>>1)] + diff = true + if (plen == qlen) && ((q + plen) < n) { + for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { + } + if j == plen { + diff = false + } + } + if diff { + name++ + q = p + qlen = plen + } + SA[m+(p>>1)] = name + } + return name +} + +func sortLMS2_{{.Type}}(T []{{.Type}}, SA, C, B, D []int, n, k int) { + var b, i, j, t, d int + var c0, c1 int + + // Compute SAl. + getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + t = 1 + } else { + t = 0 + } + j += n + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i, d = 0, 0; i < n; i++ { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) < c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + for i = n - 1; 0 <= i; i-- { + if SA[i] > 0 { + if SA[i] < n { + SA[i] += n + for j = i - 1; SA[j] < n; j-- { + } + SA[j] -= n + i = j + } + } + } + + // Compute SAs. + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i, d = n-1, d+1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) > c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + b-- + if t&1 > 0 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS2_{{.Type}}(SA []int, n, m int) int { + var i, j, d, name int + + // Compact all the sorted LMS substrings into the first m items of SA. + name = 0 + for i = 0; SA[i] < 0; i++ { + j = ^SA[i] + if n <= j { + name += 1 + } + SA[i] = j + } + if i < m { + for d, i = i, i+1; ; i++ { + if j = SA[i]; j < 0 { + j = ^j + if n <= j { + name += 1 + } + SA[d] = j + d++ + SA[i] = 0 + if d == m { + break + } + } + } + } + if name < m { + // Store the lexicographic names. + for i, d = m-1, name+1; 0 <= i; i-- { + if j = SA[i]; n <= j { + j -= n + d-- + } + SA[m+(j>>1)] = d + } + } else { + // Unset flags. + for i = 0; i < m; i++ { + if j = SA[i]; n <= j { + j -= n + SA[i] = j + } + } + } + return name +} + +func induceSA_{{.Type}}(T []{{.Type}}, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + j = SA[i] + SA[i] = ^j + if j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_{{.Type}}(T, C, n, k) + } + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + b-- + if (j == 0) || (int(T[j-1]) > c1) { + SA[b] = ^j + } else { + SA[b] = j + } + } else { + SA[i] = ^j + } + } +} + +func computeSA_{{.Type}}(T []{{.Type}}, SA []int, fs, n, k int) { + const ( + minBucketSize = 512 + sortLMS2Limit = 0x3fffffff + ) + + var C, B, D, RA []int + var bo int // Offset of B relative to SA + var b, i, j, m, p, q, name, newfs int + var c0, c1 int + var flags uint + + if k <= minBucketSize { + C = make([]int, k) + if k <= fs { + bo = n + fs - k + B = SA[bo:] + flags = 1 + } else { + B = make([]int, k) + flags = 3 + } + } else if k <= fs { + C = SA[n+fs-k:] + if k <= fs-k { + bo = n + fs - 2*k + B = SA[bo:] + flags = 0 + } else if k <= 4*minBucketSize { + B = make([]int, k) + flags = 2 + } else { + B = C + flags = 8 + } + } else { + C = make([]int, k) + B = C + flags = 4 | 8 + } + if n <= sortLMS2Limit && 2 <= (n/k) { + if flags&1 > 0 { + if 2*k <= fs-k { + flags |= 32 + } else { + flags |= 16 + } + } else if flags == 0 && 2*k <= (fs-2*k) { + flags |= 32 + } + } + + // Stage 1: Reduce the problem by at least 1/2. + // Sort all the LMS-substrings. + getCounts_{{.Type}}(T, C, n, k) + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + for i = 0; i < n; i++ { + SA[i] = 0 + } + b = -1 + i = n - 1 + j = n + m = 0 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + if b >= 0 { + SA[b] = j + } + B[c1]-- + b = B[c1] + j = i + m++ + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + if m > 1 { + if flags&(16|32) > 0 { + if flags&16 > 0 { + D = make([]int, 2*k) + } else { + D = SA[bo-2*k:] + } + B[T[j+1]]++ + for i, j = 0, 0; i < k; i++ { + j += C[i] + if B[i] != j { + SA[B[i]] += n + } + D[i] = 0 + D[i+k] = 0 + } + sortLMS2_{{.Type}}(T, SA, C, B, D, n, k) + name = postProcLMS2_{{.Type}}(SA, n, m) + } else { + sortLMS1_{{.Type}}(T, SA, C, B, n, k) + name = postProcLMS1_{{.Type}}(T, SA, n, m) + } + } else if m == 1 { + SA[b] = j + 1 + name = 1 + } else { + name = 0 + } + + // Stage 2: Solve the reduced problem. + // Recurse if names are not yet unique. + if name < m { + newfs = n + fs - 2*m + if flags&(1|4|8) == 0 { + if k+name <= newfs { + newfs -= k + } else { + flags |= 8 + } + } + RA = SA[m+newfs:] + for i, j = m+(n>>1)-1, m-1; m <= i; i-- { + if SA[i] != 0 { + RA[j] = SA[i] - 1 + j-- + } + } + computeSA_int(RA, SA, newfs, m, name) + + i = n - 1 + j = m - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + RA[j] = i + 1 + j-- + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + for i = 0; i < m; i++ { + SA[i] = RA[SA[i]] + } + if flags&4 > 0 { + B = make([]int, k) + C = B + } + if flags&2 > 0 { + B = make([]int, k) + } + } + + // Stage 3: Induce the result for the original problem. + if flags&8 > 0 { + getCounts_{{.Type}}(T, C, n, k) + } + // Put all left-most S characters into their buckets. + if m > 1 { + getBuckets_{{.Type}}(C, B, k, true) // Find ends of buckets + i = m - 1 + j = n + p = SA[m-1] + c1 = int(T[p]) + for { + c0 = c1 + q = B[c0] + for q < j { + j-- + SA[j] = 0 + } + for { + j-- + SA[j] = p + if i--; i < 0 { + break + } + p = SA[i] + if c1 = int(T[p]); c1 != c0 { + break + } + } + if i < 0 { + break + } + } + for j > 0 { + j-- + SA[j] = 0 + } + } + induceSA_{{.Type}}(T, SA, C, B, n, k) +} +` diff --git a/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go new file mode 100644 index 0000000..280682f --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/internal/sais/sais_int.go @@ -0,0 +1,661 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Code generated by sais_gen.go. DO NOT EDIT. + +// ==================================================== +// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// ==================================================== + +package sais + +func getCounts_int(T []int, C []int, n, k int) { + var i int + for i = 0; i < k; i++ { + C[i] = 0 + } + for i = 0; i < n; i++ { + C[T[i]]++ + } +} + +func getBuckets_int(C, B []int, k int, end bool) { + var i, sum int + if end { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum + } + } else { + for i = 0; i < k; i++ { + sum += C[i] + B[i] = sum - C[i] + } + } +} + +func sortLMS1_int(T []int, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + if int(T[j]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + b-- + if int(T[j]) > c1 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS1_int(T []int, SA []int, n, m int) int { + var i, j, p, q, plen, qlen, name int + var c0, c1 int + var diff bool + + // Compact all the sorted substrings into the first m items of SA. + // 2*m must be not larger than n (provable). + for i = 0; SA[i] < 0; i++ { + SA[i] = ^SA[i] + } + if i < m { + for j, i = i, i+1; ; i++ { + if p = SA[i]; p < 0 { + SA[j] = ^p + j++ + SA[i] = 0 + if j == m { + break + } + } + } + } + + // Store the length of all substrings. + i = n - 1 + j = n - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + SA[m+((i+1)>>1)] = j - i + j = i + 1 + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + // Find the lexicographic names of all substrings. + name = 0 + qlen = 0 + for i, q = 0, n; i < m; i++ { + p = SA[i] + plen = SA[m+(p>>1)] + diff = true + if (plen == qlen) && ((q + plen) < n) { + for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { + } + if j == plen { + diff = false + } + } + if diff { + name++ + q = p + qlen = plen + } + SA[m+(p>>1)] = name + } + return name +} + +func sortLMS2_int(T []int, SA, C, B, D []int, n, k int) { + var b, i, j, t, d int + var c0, c1 int + + // Compute SAl. + getBuckets_int(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + j-- + if int(T[j]) < c1 { + t = 1 + } else { + t = 0 + } + j += n + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i, d = 0, 0; i < n; i++ { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) < c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + if t&1 > 0 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + SA[i] = 0 + } else if j < 0 { + SA[i] = ^j + } + } + for i = n - 1; 0 <= i; i-- { + if SA[i] > 0 { + if SA[i] < n { + SA[i] += n + for j = i - 1; SA[j] < n; j-- { + } + SA[j] -= n + i = j + } + } + } + + // Compute SAs. + getBuckets_int(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i, d = n-1, d+1; i >= 0; i-- { + if j = SA[i]; j > 0 { + if n <= j { + d += 1 + j -= n + } + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + j-- + t = int(c0) << 1 + if int(T[j]) > c1 { + t |= 1 + } + if D[t] != d { + j += n + D[t] = d + } + b-- + if t&1 > 0 { + SA[b] = ^(j + 1) + } else { + SA[b] = j + } + SA[i] = 0 + } + } +} + +func postProcLMS2_int(SA []int, n, m int) int { + var i, j, d, name int + + // Compact all the sorted LMS substrings into the first m items of SA. + name = 0 + for i = 0; SA[i] < 0; i++ { + j = ^SA[i] + if n <= j { + name += 1 + } + SA[i] = j + } + if i < m { + for d, i = i, i+1; ; i++ { + if j = SA[i]; j < 0 { + j = ^j + if n <= j { + name += 1 + } + SA[d] = j + d++ + SA[i] = 0 + if d == m { + break + } + } + } + } + if name < m { + // Store the lexicographic names. + for i, d = m-1, name+1; 0 <= i; i-- { + if j = SA[i]; n <= j { + j -= n + d-- + } + SA[m+(j>>1)] = d + } + } else { + // Unset flags. + for i = 0; i < m; i++ { + if j = SA[i]; n <= j { + j -= n + SA[i] = j + } + } + } + return name +} + +func induceSA_int(T []int, SA, C, B []int, n, k int) { + var b, i, j int + var c0, c1 int + + // Compute SAl. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, false) // Find starts of buckets + j = n - 1 + c1 = int(T[j]) + b = B[c1] + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + for i = 0; i < n; i++ { + j = SA[i] + SA[i] = ^j + if j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + if j > 0 && int(T[j-1]) < c1 { + SA[b] = ^j + } else { + SA[b] = j + } + b++ + } + } + + // Compute SAs. + if &C[0] == &B[0] { + getCounts_int(T, C, n, k) + } + getBuckets_int(C, B, k, true) // Find ends of buckets + c1 = 0 + b = B[c1] + for i = n - 1; i >= 0; i-- { + if j = SA[i]; j > 0 { + j-- + if c0 = int(T[j]); c0 != c1 { + B[c1] = b + c1 = c0 + b = B[c1] + } + b-- + if (j == 0) || (int(T[j-1]) > c1) { + SA[b] = ^j + } else { + SA[b] = j + } + } else { + SA[i] = ^j + } + } +} + +func computeSA_int(T []int, SA []int, fs, n, k int) { + const ( + minBucketSize = 512 + sortLMS2Limit = 0x3fffffff + ) + + var C, B, D, RA []int + var bo int // Offset of B relative to SA + var b, i, j, m, p, q, name, newfs int + var c0, c1 int + var flags uint + + if k <= minBucketSize { + C = make([]int, k) + if k <= fs { + bo = n + fs - k + B = SA[bo:] + flags = 1 + } else { + B = make([]int, k) + flags = 3 + } + } else if k <= fs { + C = SA[n+fs-k:] + if k <= fs-k { + bo = n + fs - 2*k + B = SA[bo:] + flags = 0 + } else if k <= 4*minBucketSize { + B = make([]int, k) + flags = 2 + } else { + B = C + flags = 8 + } + } else { + C = make([]int, k) + B = C + flags = 4 | 8 + } + if n <= sortLMS2Limit && 2 <= (n/k) { + if flags&1 > 0 { + if 2*k <= fs-k { + flags |= 32 + } else { + flags |= 16 + } + } else if flags == 0 && 2*k <= (fs-2*k) { + flags |= 32 + } + } + + // Stage 1: Reduce the problem by at least 1/2. + // Sort all the LMS-substrings. + getCounts_int(T, C, n, k) + getBuckets_int(C, B, k, true) // Find ends of buckets + for i = 0; i < n; i++ { + SA[i] = 0 + } + b = -1 + i = n - 1 + j = n + m = 0 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + if b >= 0 { + SA[b] = j + } + B[c1]-- + b = B[c1] + j = i + m++ + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + + if m > 1 { + if flags&(16|32) > 0 { + if flags&16 > 0 { + D = make([]int, 2*k) + } else { + D = SA[bo-2*k:] + } + B[T[j+1]]++ + for i, j = 0, 0; i < k; i++ { + j += C[i] + if B[i] != j { + SA[B[i]] += n + } + D[i] = 0 + D[i+k] = 0 + } + sortLMS2_int(T, SA, C, B, D, n, k) + name = postProcLMS2_int(SA, n, m) + } else { + sortLMS1_int(T, SA, C, B, n, k) + name = postProcLMS1_int(T, SA, n, m) + } + } else if m == 1 { + SA[b] = j + 1 + name = 1 + } else { + name = 0 + } + + // Stage 2: Solve the reduced problem. + // Recurse if names are not yet unique. + if name < m { + newfs = n + fs - 2*m + if flags&(1|4|8) == 0 { + if k+name <= newfs { + newfs -= k + } else { + flags |= 8 + } + } + RA = SA[m+newfs:] + for i, j = m+(n>>1)-1, m-1; m <= i; i-- { + if SA[i] != 0 { + RA[j] = SA[i] - 1 + j-- + } + } + computeSA_int(RA, SA, newfs, m, name) + + i = n - 1 + j = m - 1 + c0 = int(T[n-1]) + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + for i >= 0 { + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 > c1 { + break + } + } + if i >= 0 { + RA[j] = i + 1 + j-- + for { + c1 = c0 + if i--; i < 0 { + break + } + if c0 = int(T[i]); c0 < c1 { + break + } + } + } + } + for i = 0; i < m; i++ { + SA[i] = RA[SA[i]] + } + if flags&4 > 0 { + B = make([]int, k) + C = B + } + if flags&2 > 0 { + B = make([]int, k) + } + } + + // Stage 3: Induce the result for the original problem. + if flags&8 > 0 { + getCounts_int(T, C, n, k) + } + // Put all left-most S characters into their buckets. + if m > 1 { + getBuckets_int(C, B, k, true) // Find ends of buckets + i = m - 1 + j = n + p = SA[m-1] + c1 = int(T[p]) + for { + c0 = c1 + q = B[c0] + for q < j { + j-- + SA[j] = 0 + } + for { + j-- + SA[j] = p + if i--; i < 0 { + break + } + p = SA[i] + if c1 = int(T[p]); c1 != c0 { + break + } + } + if i < 0 { + break + } + } + for j > 0 { + j-- + SA[j] = 0 + } + } + induceSA_int(T, SA, C, B, n, k) +} diff --git a/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go b/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go new file mode 100644 index 0000000..5c71b34 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/mtf_rle2.go @@ -0,0 +1,131 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import "github.com/dsnet/compress/internal/errors" + +// moveToFront implements both the MTF and RLE stages of bzip2 at the same time. +// Any runs of zeros in the encoded output will be replaced by a sequence of +// RUNA and RUNB symbols are encode the length of the run. +// +// The RLE encoding used can actually be encoded to and decoded from using +// normal two's complement arithmetic. The methodology for doing so is below. +// +// Assuming the following: +// num: The value being encoded by RLE encoding. +// run: A sequence of RUNA and RUNB symbols represented as a binary integer, +// where RUNA is the 0 bit, RUNB is the 1 bit, and least-significant RUN +// symbols are at the least-significant bit positions. +// cnt: The number of RUNA and RUNB symbols. +// +// Then the RLE encoding used by bzip2 has this mathematical property: +// num+1 == (1< len(mtf.dictBuf) { + panicf(errors.Internal, "alphabet too large") + } + copy(mtf.dictBuf[:], dict) + mtf.dictLen = len(dict) + mtf.blkSize = blkSize +} + +func (mtf *moveToFront) Encode(vals []byte) (syms []uint16) { + dict := mtf.dictBuf[:mtf.dictLen] + syms = mtf.syms[:0] + + if len(vals) > mtf.blkSize { + panicf(errors.Internal, "exceeded block size") + } + + var lastNum uint32 + for _, val := range vals { + // Normal move-to-front transform. + var idx uint8 // Reverse lookup idx in dict + for di, dv := range dict { + if dv == val { + idx = uint8(di) + break + } + } + copy(dict[1:], dict[:idx]) + dict[0] = val + + // Run-length encoding augmentation. + if idx == 0 { + lastNum++ + continue + } + if lastNum > 0 { + for rc := lastNum + 1; rc != 1; rc >>= 1 { + syms = append(syms, uint16(rc&1)) + } + lastNum = 0 + } + syms = append(syms, uint16(idx)+1) + } + if lastNum > 0 { + for rc := lastNum + 1; rc != 1; rc >>= 1 { + syms = append(syms, uint16(rc&1)) + } + } + mtf.syms = syms + return syms +} + +func (mtf *moveToFront) Decode(syms []uint16) (vals []byte) { + dict := mtf.dictBuf[:mtf.dictLen] + vals = mtf.vals[:0] + + var lastCnt uint + var lastRun uint32 + for _, sym := range syms { + // Run-length encoding augmentation. + if sym < 2 { + lastRun |= uint32(sym) << lastCnt + lastCnt++ + continue + } + if lastCnt > 0 { + cnt := int((1< mtf.blkSize || lastCnt > 24 { + panicf(errors.Corrupted, "run-length decoding exceeded block size") + } + for i := cnt; i > 0; i-- { + vals = append(vals, dict[0]) + } + lastCnt, lastRun = 0, 0 + } + + // Normal move-to-front transform. + val := dict[sym-1] // Forward lookup val in dict + copy(dict[1:], dict[:sym-1]) + dict[0] = val + + if len(vals) >= mtf.blkSize { + panicf(errors.Corrupted, "run-length decoding exceeded block size") + } + vals = append(vals, val) + } + if lastCnt > 0 { + cnt := int((1< mtf.blkSize || lastCnt > 24 { + panicf(errors.Corrupted, "run-length decoding exceeded block size") + } + for i := cnt; i > 0; i-- { + vals = append(vals, dict[0]) + } + } + mtf.vals = vals + return vals +} diff --git a/vendor/github.com/dsnet/compress/bzip2/prefix.go b/vendor/github.com/dsnet/compress/bzip2/prefix.go new file mode 100644 index 0000000..4847d80 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/prefix.go @@ -0,0 +1,374 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import ( + "io" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" + "github.com/dsnet/compress/internal/prefix" +) + +const ( + minNumTrees = 2 + maxNumTrees = 6 + + maxPrefixBits = 20 // Maximum bit-width of a prefix code + maxNumSyms = 256 + 2 // Maximum number of symbols in the alphabet + numBlockSyms = 50 // Number of bytes in a block +) + +// encSel and decSel are used to handle the prefix encoding for tree selectors. +// The prefix encoding is as follows: +// +// Code TreeIdx +// 0 <=> 0 +// 10 <=> 1 +// 110 <=> 2 +// 1110 <=> 3 +// 11110 <=> 4 +// 111110 <=> 5 +// 111111 <=> 6 Invalid tree index, so should fail +// +var encSel, decSel = func() (e prefix.Encoder, d prefix.Decoder) { + var selCodes [maxNumTrees + 1]prefix.PrefixCode + for i := range selCodes { + selCodes[i] = prefix.PrefixCode{Sym: uint32(i), Len: uint32(i + 1)} + } + selCodes[maxNumTrees] = prefix.PrefixCode{Sym: maxNumTrees, Len: maxNumTrees} + prefix.GeneratePrefixes(selCodes[:]) + e.Init(selCodes[:]) + d.Init(selCodes[:]) + return +}() + +type prefixReader struct{ prefix.Reader } + +func (pr *prefixReader) Init(r io.Reader) { + pr.Reader.Init(r, true) +} + +func (pr *prefixReader) ReadBitsBE64(nb uint) uint64 { + if nb <= 32 { + v := uint32(pr.ReadBits(nb)) + return uint64(internal.ReverseUint32N(v, nb)) + } + v0 := internal.ReverseUint32(uint32(pr.ReadBits(32))) + v1 := internal.ReverseUint32(uint32(pr.ReadBits(nb - 32))) + v := uint64(v0)<<32 | uint64(v1) + return v >> (64 - nb) +} + +func (pr *prefixReader) ReadPrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Decoder) { + for i, pc := range codes { + clen := int(pr.ReadBitsBE64(5)) + sum := 1 << maxPrefixBits + for sym := range pc { + for { + if clen < 1 || clen > maxPrefixBits { + panicf(errors.Corrupted, "invalid prefix bit-length: %d", clen) + } + + b, ok := pr.TryReadBits(1) + if !ok { + b = pr.ReadBits(1) + } + if b == 0 { + break + } + + b, ok = pr.TryReadBits(1) + if !ok { + b = pr.ReadBits(1) + } + clen -= int(b*2) - 1 // +1 or -1 + } + pc[sym] = prefix.PrefixCode{Sym: uint32(sym), Len: uint32(clen)} + sum -= (1 << maxPrefixBits) >> uint(clen) + } + + if sum == 0 { + // Fast path, but only handles complete trees. + if err := prefix.GeneratePrefixes(pc); err != nil { + errors.Panic(err) // Using complete trees; should never fail + } + } else { + // Slow path, but handles anything. + pc = handleDegenerateCodes(pc) // Never fails, but may fail later + codes[i] = pc + } + trees[i].Init(pc) + } +} + +type prefixWriter struct{ prefix.Writer } + +func (pw *prefixWriter) Init(w io.Writer) { + pw.Writer.Init(w, true) +} + +func (pw *prefixWriter) WriteBitsBE64(v uint64, nb uint) { + if nb <= 32 { + v := internal.ReverseUint32N(uint32(v), nb) + pw.WriteBits(uint(v), nb) + return + } + v <<= (64 - nb) + v0 := internal.ReverseUint32(uint32(v >> 32)) + v1 := internal.ReverseUint32(uint32(v)) + pw.WriteBits(uint(v0), 32) + pw.WriteBits(uint(v1), nb-32) + return +} + +func (pw *prefixWriter) WritePrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Encoder) { + for i, pc := range codes { + if err := prefix.GeneratePrefixes(pc); err != nil { + errors.Panic(err) // Using complete trees; should never fail + } + trees[i].Init(pc) + + clen := int(pc[0].Len) + pw.WriteBitsBE64(uint64(clen), 5) + for _, c := range pc { + for int(c.Len) < clen { + pw.WriteBits(3, 2) // 11 + clen-- + } + for int(c.Len) > clen { + pw.WriteBits(1, 2) // 10 + clen++ + } + pw.WriteBits(0, 1) + } + } +} + +// handleDegenerateCodes converts a degenerate tree into a canonical tree. +// +// For example, when the input is an under-subscribed tree: +// input: []PrefixCode{ +// {Sym: 0, Len: 3}, +// {Sym: 1, Len: 4}, +// {Sym: 2, Len: 3}, +// } +// output: []PrefixCode{ +// {Sym: 0, Len: 3, Val: 0}, // 000 +// {Sym: 1, Len: 4, Val: 2}, // 0010 +// {Sym: 2, Len: 3, Val: 4}, // 100 +// {Sym: 258, Len: 4, Val: 10}, // 1010 +// {Sym: 259, Len: 3, Val: 6}, // 110 +// {Sym: 260, Len: 1, Val: 1}, // 1 +// } +// +// For example, when the input is an over-subscribed tree: +// input: []PrefixCode{ +// {Sym: 0, Len: 1}, +// {Sym: 1, Len: 3}, +// {Sym: 2, Len: 4}, +// {Sym: 3, Len: 3}, +// {Sym: 4, Len: 2}, +// } +// output: []PrefixCode{ +// {Sym: 0, Len: 1, Val: 0}, // 0 +// {Sym: 1, Len: 3, Val: 3}, // 011 +// {Sym: 3, Len: 3, Val: 7}, // 111 +// {Sym: 4, Len: 2, Val: 1}, // 01 +// } +func handleDegenerateCodes(codes prefix.PrefixCodes) prefix.PrefixCodes { + // Since there is no formal definition for the BZip2 format, there is no + // specification that says that the code lengths must form a complete + // prefix tree (IE: it is neither over-subscribed nor under-subscribed). + // Thus, the original C implementation becomes the reference for how prefix + // decoding is done in these edge cases. Unfortunately, the C version does + // not error when an invalid tree is used, but rather allows decoding to + // continue and only errors if some bit pattern happens to cause an error. + // Thus, it is possible for an invalid tree to end up decoding an input + // "properly" so long as invalid bit patterns are not present. In order to + // replicate this non-specified behavior, we use a ported version of the + // C code to generate the codes as a valid canonical tree by substituting + // invalid nodes with invalid symbols. + // + // ==================================================== + // This program, "bzip2", the associated library "libbzip2", and all + // documentation, are copyright (C) 1996-2010 Julian R Seward. All + // rights reserved. + // + // Redistribution and use in source and binary forms, with or without + // modification, are permitted provided that the following conditions + // are met: + // + // 1. Redistributions of source code must retain the above copyright + // notice, this list of conditions and the following disclaimer. + // + // 2. The origin of this software must not be misrepresented; you must + // not claim that you wrote the original software. If you use this + // software in a product, an acknowledgment in the product + // documentation would be appreciated but is not required. + // + // 3. Altered source versions must be plainly marked as such, and must + // not be misrepresented as being the original software. + // + // 4. The name of the author may not be used to endorse or promote + // products derived from this software without specific prior written + // permission. + // + // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + // OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + // ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + // + // Julian Seward, jseward@bzip.org + // bzip2/libbzip2 version 1.0.6 of 6 September 2010 + // ==================================================== + var ( + limits [maxPrefixBits + 2]int32 + bases [maxPrefixBits + 2]int32 + perms [maxNumSyms]int32 + + minLen = uint32(maxPrefixBits) + maxLen = uint32(0) + ) + + const ( + statusOkay = iota + statusInvalid + statusNeedBits + statusMaxBits + ) + + // createTables is the BZ2_hbCreateDecodeTables function from the C code. + createTables := func(codes []prefix.PrefixCode) { + for _, c := range codes { + if c.Len > maxLen { + maxLen = c.Len + } + if c.Len < minLen { + minLen = c.Len + } + } + + var pp int + for i := minLen; i <= maxLen; i++ { + for j, c := range codes { + if c.Len == i { + perms[pp] = int32(j) + pp++ + } + } + } + + var vec int32 + for _, c := range codes { + bases[c.Len+1]++ + } + for i := 1; i < len(bases); i++ { + bases[i] += bases[i-1] + } + for i := minLen; i <= maxLen; i++ { + vec += bases[i+1] - bases[i] + limits[i] = vec - 1 + vec <<= 1 + } + for i := minLen + 1; i <= maxLen; i++ { + bases[i] = ((limits[i-1] + 1) << 1) - bases[i] + } + } + + // getSymbol is the GET_MTF_VAL macro from the C code. + getSymbol := func(c prefix.PrefixCode) (uint32, int) { + v := internal.ReverseUint32(c.Val) + n := c.Len + + zn := minLen + if zn > n { + return 0, statusNeedBits + } + zvec := int32(v >> (32 - zn)) + v <<= zn + for { + if zn > maxLen { + return 0, statusMaxBits + } + if zvec <= limits[zn] { + break + } + zn++ + if zn > n { + return 0, statusNeedBits + } + zvec = (zvec << 1) | int32(v>>31) + v <<= 1 + } + if zvec-bases[zn] < 0 || zvec-bases[zn] >= maxNumSyms { + return 0, statusInvalid + } + return uint32(perms[zvec-bases[zn]]), statusOkay + } + + // Step 1: Create the prefix trees using the C algorithm. + createTables(codes) + + // Step 2: Starting with the shortest bit pattern, explore the whole tree. + // If tree is under-subscribed, the worst-case runtime is O(1< 0 { + codes = append(codes, c) + } + } + return codes +} diff --git a/vendor/github.com/dsnet/compress/bzip2/reader.go b/vendor/github.com/dsnet/compress/bzip2/reader.go new file mode 100644 index 0000000..86d3f71 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/reader.go @@ -0,0 +1,274 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import ( + "io" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" + "github.com/dsnet/compress/internal/prefix" +) + +type Reader struct { + InputOffset int64 // Total number of bytes read from underlying io.Reader + OutputOffset int64 // Total number of bytes emitted from Read + + rd prefixReader + err error + level int // The current compression level + rdHdrFtr int // Number of times we read the stream header and footer + blkCRC uint32 // CRC-32 IEEE of each block (as stored) + endCRC uint32 // Checksum of all blocks using bzip2's custom method + + crc crc + mtf moveToFront + bwt burrowsWheelerTransform + rle runLengthEncoding + + // These fields are allocated with Reader and re-used later. + treeSels []uint8 + codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode + codes1D [maxNumTrees]prefix.PrefixCodes + trees1D [maxNumTrees]prefix.Decoder + syms []uint16 + + fuzzReader // Exported functionality when fuzz testing +} + +type ReaderConfig struct { + _ struct{} // Blank field to prevent unkeyed struct literals +} + +func NewReader(r io.Reader, conf *ReaderConfig) (*Reader, error) { + zr := new(Reader) + zr.Reset(r) + return zr, nil +} + +func (zr *Reader) Reset(r io.Reader) error { + *zr = Reader{ + rd: zr.rd, + + mtf: zr.mtf, + bwt: zr.bwt, + rle: zr.rle, + + treeSels: zr.treeSels, + trees1D: zr.trees1D, + syms: zr.syms, + } + zr.rd.Init(r) + return nil +} + +func (zr *Reader) Read(buf []byte) (int, error) { + for { + cnt, err := zr.rle.Read(buf) + if err != rleDone && zr.err == nil { + zr.err = err + } + if cnt > 0 { + zr.crc.update(buf[:cnt]) + zr.OutputOffset += int64(cnt) + return cnt, nil + } + if zr.err != nil || len(buf) == 0 { + return 0, zr.err + } + + // Read the next chunk. + zr.rd.Offset = zr.InputOffset + func() { + defer errors.Recover(&zr.err) + if zr.rdHdrFtr%2 == 0 { + // Check if we are already at EOF. + if err := zr.rd.PullBits(1); err != nil { + if err == io.ErrUnexpectedEOF && zr.rdHdrFtr > 0 { + err = io.EOF // EOF is okay if we read at least one stream + } + errors.Panic(err) + } + + // Read stream header. + if zr.rd.ReadBitsBE64(16) != hdrMagic { + panicf(errors.Corrupted, "invalid stream magic") + } + if ver := zr.rd.ReadBitsBE64(8); ver != 'h' { + if ver == '0' { + panicf(errors.Deprecated, "bzip1 format is not supported") + } + panicf(errors.Corrupted, "invalid version: %q", ver) + } + lvl := int(zr.rd.ReadBitsBE64(8)) - '0' + if lvl < BestSpeed || lvl > BestCompression { + panicf(errors.Corrupted, "invalid block size: %d", lvl*blockSize) + } + zr.level = lvl + zr.rdHdrFtr++ + } else { + // Check and update the CRC. + if internal.GoFuzz { + zr.updateChecksum(-1, zr.crc.val) // Update with value + zr.blkCRC = zr.crc.val // Suppress CRC failures + } + if zr.blkCRC != zr.crc.val { + panicf(errors.Corrupted, "mismatching block checksum") + } + zr.endCRC = (zr.endCRC<<1 | zr.endCRC>>31) ^ zr.blkCRC + } + buf := zr.decodeBlock() + zr.rle.Init(buf) + }() + if zr.InputOffset, err = zr.rd.Flush(); zr.err == nil { + zr.err = err + } + if zr.err != nil { + zr.err = errWrap(zr.err, errors.Corrupted) + return 0, zr.err + } + } +} + +func (zr *Reader) Close() error { + if zr.err == io.EOF || zr.err == errClosed { + zr.rle.Init(nil) // Make sure future reads fail + zr.err = errClosed + return nil + } + return zr.err // Return the persistent error +} + +func (zr *Reader) decodeBlock() []byte { + if magic := zr.rd.ReadBitsBE64(48); magic != blkMagic { + if magic == endMagic { + endCRC := uint32(zr.rd.ReadBitsBE64(32)) + if internal.GoFuzz { + zr.updateChecksum(zr.rd.BitsRead()-32, zr.endCRC) + endCRC = zr.endCRC // Suppress CRC failures + } + if zr.endCRC != endCRC { + panicf(errors.Corrupted, "mismatching stream checksum") + } + zr.endCRC = 0 + zr.rd.ReadPads() + zr.rdHdrFtr++ + return nil + } + panicf(errors.Corrupted, "invalid block or footer magic") + } + + zr.crc.val = 0 + zr.blkCRC = uint32(zr.rd.ReadBitsBE64(32)) + if internal.GoFuzz { + zr.updateChecksum(zr.rd.BitsRead()-32, 0) // Record offset only + } + if zr.rd.ReadBitsBE64(1) != 0 { + panicf(errors.Deprecated, "block randomization is not supported") + } + + // Read BWT related fields. + ptr := int(zr.rd.ReadBitsBE64(24)) // BWT origin pointer + + // Read MTF related fields. + var dictArr [256]uint8 + dict := dictArr[:0] + bmapHi := uint16(zr.rd.ReadBits(16)) + for i := 0; i < 256; i, bmapHi = i+16, bmapHi>>1 { + if bmapHi&1 > 0 { + bmapLo := uint16(zr.rd.ReadBits(16)) + for j := 0; j < 16; j, bmapLo = j+1, bmapLo>>1 { + if bmapLo&1 > 0 { + dict = append(dict, uint8(i+j)) + } + } + } + } + + // Step 1: Prefix encoding. + syms := zr.decodePrefix(len(dict)) + + // Step 2: Move-to-front transform and run-length encoding. + zr.mtf.Init(dict, zr.level*blockSize) + buf := zr.mtf.Decode(syms) + + // Step 3: Burrows-Wheeler transformation. + if ptr >= len(buf) { + panicf(errors.Corrupted, "origin pointer (0x%06x) exceeds block size: %d", ptr, len(buf)) + } + zr.bwt.Decode(buf, ptr) + + return buf +} + +func (zr *Reader) decodePrefix(numSyms int) (syms []uint16) { + numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOF symbols + if numSyms < 3 { + panicf(errors.Corrupted, "not enough prefix symbols: %d", numSyms) + } + + // Read information about the trees and tree selectors. + var mtf internal.MoveToFront + numTrees := int(zr.rd.ReadBitsBE64(3)) + if numTrees < minNumTrees || numTrees > maxNumTrees { + panicf(errors.Corrupted, "invalid number of prefix trees: %d", numTrees) + } + numSels := int(zr.rd.ReadBitsBE64(15)) + if cap(zr.treeSels) < numSels { + zr.treeSels = make([]uint8, numSels) + } + treeSels := zr.treeSels[:numSels] + for i := range treeSels { + sym, ok := zr.rd.TryReadSymbol(&decSel) + if !ok { + sym = zr.rd.ReadSymbol(&decSel) + } + if int(sym) >= numTrees { + panicf(errors.Corrupted, "invalid prefix tree selector: %d", sym) + } + treeSels[i] = uint8(sym) + } + mtf.Decode(treeSels) + zr.treeSels = treeSels + + // Initialize prefix codes. + for i := range zr.codes2D[:numTrees] { + zr.codes1D[i] = zr.codes2D[i][:numSyms] + } + zr.rd.ReadPrefixCodes(zr.codes1D[:numTrees], zr.trees1D[:numTrees]) + + // Read prefix encoded symbols of compressed data. + var tree *prefix.Decoder + var blkLen, selIdx int + syms = zr.syms[:0] + for { + if blkLen == 0 { + blkLen = numBlockSyms + if selIdx >= len(treeSels) { + panicf(errors.Corrupted, "not enough prefix tree selectors") + } + tree = &zr.trees1D[treeSels[selIdx]] + selIdx++ + } + blkLen-- + sym, ok := zr.rd.TryReadSymbol(tree) + if !ok { + sym = zr.rd.ReadSymbol(tree) + } + + if int(sym) == numSyms-1 { + break // EOF marker + } + if int(sym) >= numSyms { + panicf(errors.Corrupted, "invalid prefix symbol: %d", sym) + } + if len(syms) >= zr.level*blockSize { + panicf(errors.Corrupted, "number of prefix symbols exceeds block size") + } + syms = append(syms, uint16(sym)) + } + zr.syms = syms + return syms +} diff --git a/vendor/github.com/dsnet/compress/bzip2/rle1.go b/vendor/github.com/dsnet/compress/bzip2/rle1.go new file mode 100644 index 0000000..1d789f6 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/rle1.go @@ -0,0 +1,101 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import "github.com/dsnet/compress/internal/errors" + +// rleDone is a special "error" to indicate that the RLE stage is done. +var rleDone = errorf(errors.Unknown, "RLE1 stage is completed") + +// runLengthEncoding implements the first RLE stage of bzip2. Every sequence +// of 4..255 duplicated bytes is replaced by only the first 4 bytes, and a +// single byte representing the repeat length. Similar to the C bzip2 +// implementation, the encoder will always terminate repeat sequences with a +// count (even if it is the end of the buffer), and it will also never produce +// run lengths of 256..259. The decoder can handle the latter case. +// +// For example, if the input was: +// input: "AAAAAAABBBBCCCD" +// +// Then the output will be: +// output: "AAAA\x03BBBB\x00CCCD" +type runLengthEncoding struct { + buf []byte + idx int + lastVal byte + lastCnt int +} + +func (rle *runLengthEncoding) Init(buf []byte) { + *rle = runLengthEncoding{buf: buf} +} + +func (rle *runLengthEncoding) Write(buf []byte) (int, error) { + for i, b := range buf { + if rle.lastVal != b { + rle.lastCnt = 0 + } + rle.lastCnt++ + switch { + case rle.lastCnt < 4: + if rle.idx >= len(rle.buf) { + return i, rleDone + } + rle.buf[rle.idx] = b + rle.idx++ + case rle.lastCnt == 4: + if rle.idx+1 >= len(rle.buf) { + return i, rleDone + } + rle.buf[rle.idx] = b + rle.idx++ + rle.buf[rle.idx] = 0 + rle.idx++ + case rle.lastCnt < 256: + rle.buf[rle.idx-1]++ + default: + if rle.idx >= len(rle.buf) { + return i, rleDone + } + rle.lastCnt = 1 + rle.buf[rle.idx] = b + rle.idx++ + } + rle.lastVal = b + } + return len(buf), nil +} + +func (rle *runLengthEncoding) Read(buf []byte) (int, error) { + for i := range buf { + switch { + case rle.lastCnt == -4: + if rle.idx >= len(rle.buf) { + return i, errorf(errors.Corrupted, "missing terminating run-length repeater") + } + rle.lastCnt = int(rle.buf[rle.idx]) + rle.idx++ + if rle.lastCnt > 0 { + break // Break the switch + } + fallthrough // Count was zero, continue the work + case rle.lastCnt <= 0: + if rle.idx >= len(rle.buf) { + return i, rleDone + } + b := rle.buf[rle.idx] + rle.idx++ + if b != rle.lastVal { + rle.lastCnt = 0 + rle.lastVal = b + } + } + buf[i] = rle.lastVal + rle.lastCnt-- + } + return len(buf), nil +} + +func (rle *runLengthEncoding) Bytes() []byte { return rle.buf[:rle.idx] } diff --git a/vendor/github.com/dsnet/compress/bzip2/writer.go b/vendor/github.com/dsnet/compress/bzip2/writer.go new file mode 100644 index 0000000..5c1a4c6 --- /dev/null +++ b/vendor/github.com/dsnet/compress/bzip2/writer.go @@ -0,0 +1,307 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package bzip2 + +import ( + "io" + + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" + "github.com/dsnet/compress/internal/prefix" +) + +type Writer struct { + InputOffset int64 // Total number of bytes issued to Write + OutputOffset int64 // Total number of bytes written to underlying io.Writer + + wr prefixWriter + err error + level int // The current compression level + wrHdr bool // Have we written the stream header? + blkCRC uint32 // CRC-32 IEEE of each block + endCRC uint32 // Checksum of all blocks using bzip2's custom method + + crc crc + rle runLengthEncoding + bwt burrowsWheelerTransform + mtf moveToFront + + // These fields are allocated with Writer and re-used later. + buf []byte + treeSels []uint8 + treeSelsMTF []uint8 + codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode + codes1D [maxNumTrees]prefix.PrefixCodes + trees1D [maxNumTrees]prefix.Encoder +} + +type WriterConfig struct { + Level int + + _ struct{} // Blank field to prevent unkeyed struct literals +} + +func NewWriter(w io.Writer, conf *WriterConfig) (*Writer, error) { + var lvl int + if conf != nil { + lvl = conf.Level + } + if lvl == 0 { + lvl = DefaultCompression + } + if lvl < BestSpeed || lvl > BestCompression { + return nil, errorf(errors.Invalid, "compression level: %d", lvl) + } + zw := new(Writer) + zw.level = lvl + zw.Reset(w) + return zw, nil +} + +func (zw *Writer) Reset(w io.Writer) error { + *zw = Writer{ + wr: zw.wr, + level: zw.level, + + rle: zw.rle, + bwt: zw.bwt, + mtf: zw.mtf, + + buf: zw.buf, + treeSels: zw.treeSels, + treeSelsMTF: zw.treeSelsMTF, + trees1D: zw.trees1D, + } + zw.wr.Init(w) + if len(zw.buf) != zw.level*blockSize { + zw.buf = make([]byte, zw.level*blockSize) + } + zw.rle.Init(zw.buf) + return nil +} + +func (zw *Writer) Write(buf []byte) (int, error) { + if zw.err != nil { + return 0, zw.err + } + + cnt := len(buf) + for { + wrCnt, err := zw.rle.Write(buf) + if err != rleDone && zw.err == nil { + zw.err = err + } + zw.crc.update(buf[:wrCnt]) + buf = buf[wrCnt:] + if len(buf) == 0 { + zw.InputOffset += int64(cnt) + return cnt, nil + } + if zw.err = zw.flush(); zw.err != nil { + return 0, zw.err + } + } +} + +func (zw *Writer) flush() error { + vals := zw.rle.Bytes() + if len(vals) == 0 { + return nil + } + zw.wr.Offset = zw.OutputOffset + func() { + defer errors.Recover(&zw.err) + if !zw.wrHdr { + // Write stream header. + zw.wr.WriteBitsBE64(hdrMagic, 16) + zw.wr.WriteBitsBE64('h', 8) + zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) + zw.wrHdr = true + } + zw.encodeBlock(vals) + }() + var err error + if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { + zw.err = err + } + if zw.err != nil { + zw.err = errWrap(zw.err, errors.Internal) + return zw.err + } + zw.endCRC = (zw.endCRC<<1 | zw.endCRC>>31) ^ zw.blkCRC + zw.blkCRC = 0 + zw.rle.Init(zw.buf) + return nil +} + +func (zw *Writer) Close() error { + if zw.err == errClosed { + return nil + } + + // Flush RLE buffer if there is left-over data. + if zw.err = zw.flush(); zw.err != nil { + return zw.err + } + + // Write stream footer. + zw.wr.Offset = zw.OutputOffset + func() { + defer errors.Recover(&zw.err) + if !zw.wrHdr { + // Write stream header. + zw.wr.WriteBitsBE64(hdrMagic, 16) + zw.wr.WriteBitsBE64('h', 8) + zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) + zw.wrHdr = true + } + zw.wr.WriteBitsBE64(endMagic, 48) + zw.wr.WriteBitsBE64(uint64(zw.endCRC), 32) + zw.wr.WritePads(0) + }() + var err error + if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { + zw.err = err + } + if zw.err != nil { + zw.err = errWrap(zw.err, errors.Internal) + return zw.err + } + + zw.err = errClosed + return nil +} + +func (zw *Writer) encodeBlock(buf []byte) { + zw.blkCRC = zw.crc.val + zw.wr.WriteBitsBE64(blkMagic, 48) + zw.wr.WriteBitsBE64(uint64(zw.blkCRC), 32) + zw.wr.WriteBitsBE64(0, 1) + zw.crc.val = 0 + + // Step 1: Burrows-Wheeler transformation. + ptr := zw.bwt.Encode(buf) + zw.wr.WriteBitsBE64(uint64(ptr), 24) + + // Step 2: Move-to-front transform and run-length encoding. + var dictMap [256]bool + for _, c := range buf { + dictMap[c] = true + } + + var dictArr [256]uint8 + var bmapLo [16]uint16 + dict := dictArr[:0] + bmapHi := uint16(0) + for i, b := range dictMap { + if b { + c := uint8(i) + dict = append(dict, c) + bmapHi |= 1 << (c >> 4) + bmapLo[c>>4] |= 1 << (c & 0xf) + } + } + + zw.wr.WriteBits(uint(bmapHi), 16) + for _, m := range bmapLo { + if m > 0 { + zw.wr.WriteBits(uint(m), 16) + } + } + + zw.mtf.Init(dict, len(buf)) + syms := zw.mtf.Encode(buf) + + // Step 3: Prefix encoding. + zw.encodePrefix(syms, len(dict)) +} + +func (zw *Writer) encodePrefix(syms []uint16, numSyms int) { + numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOB symbols + if numSyms < 3 { + panicf(errors.Internal, "unable to encode EOB marker") + } + syms = append(syms, uint16(numSyms-1)) // EOB marker + + // Compute number of prefix trees needed. + numTrees := maxNumTrees + for i, lim := range []int{200, 600, 1200, 2400} { + if len(syms) < lim { + numTrees = minNumTrees + i + break + } + } + + // Compute number of block selectors. + numSels := (len(syms) + numBlockSyms - 1) / numBlockSyms + if cap(zw.treeSels) < numSels { + zw.treeSels = make([]uint8, numSels) + } + treeSels := zw.treeSels[:numSels] + for i := range treeSels { + treeSels[i] = uint8(i % numTrees) + } + + // Initialize prefix codes. + for i := range zw.codes2D[:numTrees] { + pc := zw.codes2D[i][:numSyms] + for j := range pc { + pc[j] = prefix.PrefixCode{Sym: uint32(j)} + } + zw.codes1D[i] = pc + } + + // First cut at assigning prefix trees to each group. + var codes prefix.PrefixCodes + var blkLen, selIdx int + for _, sym := range syms { + if blkLen == 0 { + blkLen = numBlockSyms + codes = zw.codes2D[treeSels[selIdx]][:numSyms] + selIdx++ + } + blkLen-- + codes[sym].Cnt++ + } + + // TODO(dsnet): Use K-means to cluster groups to each prefix tree. + + // Generate lengths and prefixes based on symbol frequencies. + for i := range zw.trees1D[:numTrees] { + pc := prefix.PrefixCodes(zw.codes2D[i][:numSyms]) + pc.SortByCount() + if err := prefix.GenerateLengths(pc, maxPrefixBits); err != nil { + errors.Panic(err) + } + pc.SortBySymbol() + } + + // Write out information about the trees and tree selectors. + var mtf internal.MoveToFront + zw.wr.WriteBitsBE64(uint64(numTrees), 3) + zw.wr.WriteBitsBE64(uint64(numSels), 15) + zw.treeSelsMTF = append(zw.treeSelsMTF[:0], treeSels...) + mtf.Encode(zw.treeSelsMTF) + for _, sym := range zw.treeSelsMTF { + zw.wr.WriteSymbol(uint(sym), &encSel) + } + zw.wr.WritePrefixCodes(zw.codes1D[:numTrees], zw.trees1D[:numTrees]) + + // Write out prefix encoded symbols of compressed data. + var tree *prefix.Encoder + blkLen, selIdx = 0, 0 + for _, sym := range syms { + if blkLen == 0 { + blkLen = numBlockSyms + tree = &zw.trees1D[treeSels[selIdx]] + selIdx++ + } + blkLen-- + ok := zw.wr.TryWriteSymbol(uint(sym), tree) + if !ok { + zw.wr.WriteSymbol(uint(sym), tree) + } + } +} diff --git a/vendor/github.com/dsnet/compress/internal/common.go b/vendor/github.com/dsnet/compress/internal/common.go new file mode 100644 index 0000000..da4e703 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/common.go @@ -0,0 +1,107 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package internal is a collection of common compression algorithms. +// +// For performance reasons, these packages lack strong error checking and +// require that the caller to ensure that strict invariants are kept. +package internal + +var ( + // IdentityLUT returns the input key itself. + IdentityLUT = func() (lut [256]byte) { + for i := range lut { + lut[i] = uint8(i) + } + return lut + }() + + // ReverseLUT returns the input key with its bits reversed. + ReverseLUT = func() (lut [256]byte) { + for i := range lut { + b := uint8(i) + b = (b&0xaa)>>1 | (b&0x55)<<1 + b = (b&0xcc)>>2 | (b&0x33)<<2 + b = (b&0xf0)>>4 | (b&0x0f)<<4 + lut[i] = b + } + return lut + }() +) + +// ReverseUint32 reverses all bits of v. +func ReverseUint32(v uint32) (x uint32) { + x |= uint32(ReverseLUT[byte(v>>0)]) << 24 + x |= uint32(ReverseLUT[byte(v>>8)]) << 16 + x |= uint32(ReverseLUT[byte(v>>16)]) << 8 + x |= uint32(ReverseLUT[byte(v>>24)]) << 0 + return x +} + +// ReverseUint32N reverses the lower n bits of v. +func ReverseUint32N(v uint32, n uint) (x uint32) { + return ReverseUint32(v << (32 - n)) +} + +// ReverseUint64 reverses all bits of v. +func ReverseUint64(v uint64) (x uint64) { + x |= uint64(ReverseLUT[byte(v>>0)]) << 56 + x |= uint64(ReverseLUT[byte(v>>8)]) << 48 + x |= uint64(ReverseLUT[byte(v>>16)]) << 40 + x |= uint64(ReverseLUT[byte(v>>24)]) << 32 + x |= uint64(ReverseLUT[byte(v>>32)]) << 24 + x |= uint64(ReverseLUT[byte(v>>40)]) << 16 + x |= uint64(ReverseLUT[byte(v>>48)]) << 8 + x |= uint64(ReverseLUT[byte(v>>56)]) << 0 + return x +} + +// ReverseUint64N reverses the lower n bits of v. +func ReverseUint64N(v uint64, n uint) (x uint64) { + return ReverseUint64(v << (64 - n)) +} + +// MoveToFront is a data structure that allows for more efficient move-to-front +// transformations. This specific implementation assumes that the alphabet is +// densely packed within 0..255. +type MoveToFront struct { + dict [256]uint8 // Mapping from indexes to values + tail int // Number of tail bytes that are already ordered +} + +func (m *MoveToFront) Encode(vals []uint8) { + copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity + + var max int + for i, val := range vals { + var idx uint8 // Reverse lookup idx in dict + for di, dv := range m.dict { + if dv == val { + idx = uint8(di) + break + } + } + vals[i] = idx + + max |= int(idx) + copy(m.dict[1:], m.dict[:idx]) + m.dict[0] = val + } + m.tail = 256 - max - 1 +} + +func (m *MoveToFront) Decode(idxs []uint8) { + copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity + + var max int + for i, idx := range idxs { + val := m.dict[idx] // Forward lookup val in dict + idxs[i] = val + + max |= int(idx) + copy(m.dict[1:], m.dict[:idx]) + m.dict[0] = val + } + m.tail = 256 - max - 1 +} diff --git a/vendor/github.com/dsnet/compress/internal/debug.go b/vendor/github.com/dsnet/compress/internal/debug.go new file mode 100644 index 0000000..01df1f8 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/debug.go @@ -0,0 +1,12 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build debug,!gofuzz + +package internal + +const ( + Debug = true + GoFuzz = false +) diff --git a/vendor/github.com/dsnet/compress/internal/errors/errors.go b/vendor/github.com/dsnet/compress/internal/errors/errors.go new file mode 100644 index 0000000..c631afb --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/errors/errors.go @@ -0,0 +1,120 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// Package errors implements functions to manipulate compression errors. +// +// In idiomatic Go, it is an anti-pattern to use panics as a form of error +// reporting in the API. Instead, the expected way to transmit errors is by +// returning an error value. Unfortunately, the checking of "err != nil" in +// tight loops commonly found in compression causes non-negligible performance +// degradation. While this may not be idiomatic, the internal packages of this +// repository rely on panics as a normal means to convey errors. In order to +// ensure that these panics do not leak across the public API, the public +// packages must recover from these panics and present an error value. +// +// The Panic and Recover functions in this package provide a safe way to +// recover from errors only generated from within this repository. +// +// Example usage: +// func Foo() (err error) { +// defer errors.Recover(&err) +// +// if rand.Intn(2) == 0 { +// // Unexpected panics will not be caught by Recover. +// io.Closer(nil).Close() +// } else { +// // Errors thrown by Panic will be caught by Recover. +// errors.Panic(errors.New("whoopsie")) +// } +// } +// +package errors + +import "strings" + +const ( + // Unknown indicates that there is no classification for this error. + Unknown = iota + + // Internal indicates that this error is due to an internal bug. + // Users should file a issue report if this type of error is encountered. + Internal + + // Invalid indicates that this error is due to the user misusing the API + // and is indicative of a bug on the user's part. + Invalid + + // Deprecated indicates the use of a deprecated and unsupported feature. + Deprecated + + // Corrupted indicates that the input stream is corrupted. + Corrupted + + // Closed indicates that the handlers are closed. + Closed +) + +var codeMap = map[int]string{ + Unknown: "unknown error", + Internal: "internal error", + Invalid: "invalid argument", + Deprecated: "deprecated format", + Corrupted: "corrupted input", + Closed: "closed handler", +} + +type Error struct { + Code int // The error type + Pkg string // Name of the package where the error originated + Msg string // Descriptive message about the error (optional) +} + +func (e Error) Error() string { + var ss []string + for _, s := range []string{e.Pkg, codeMap[e.Code], e.Msg} { + if s != "" { + ss = append(ss, s) + } + } + return strings.Join(ss, ": ") +} + +func (e Error) CompressError() {} +func (e Error) IsInternal() bool { return e.Code == Internal } +func (e Error) IsInvalid() bool { return e.Code == Invalid } +func (e Error) IsDeprecated() bool { return e.Code == Deprecated } +func (e Error) IsCorrupted() bool { return e.Code == Corrupted } +func (e Error) IsClosed() bool { return e.Code == Closed } + +func IsInternal(err error) bool { return isCode(err, Internal) } +func IsInvalid(err error) bool { return isCode(err, Invalid) } +func IsDeprecated(err error) bool { return isCode(err, Deprecated) } +func IsCorrupted(err error) bool { return isCode(err, Corrupted) } +func IsClosed(err error) bool { return isCode(err, Closed) } + +func isCode(err error, code int) bool { + if cerr, ok := err.(Error); ok && cerr.Code == code { + return true + } + return false +} + +// errWrap is used by Panic and Recover to ensure that only errors raised by +// Panic are recovered by Recover. +type errWrap struct{ e *error } + +func Recover(err *error) { + switch ex := recover().(type) { + case nil: + // Do nothing. + case errWrap: + *err = *ex.e + default: + panic(ex) + } +} + +func Panic(err error) { + panic(errWrap{&err}) +} diff --git a/vendor/github.com/dsnet/compress/internal/gofuzz.go b/vendor/github.com/dsnet/compress/internal/gofuzz.go new file mode 100644 index 0000000..5035c9d --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/gofuzz.go @@ -0,0 +1,12 @@ +// Copyright 2016, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build gofuzz + +package internal + +const ( + Debug = true + GoFuzz = true +) diff --git a/vendor/github.com/dsnet/compress/internal/prefix/debug.go b/vendor/github.com/dsnet/compress/internal/prefix/debug.go new file mode 100644 index 0000000..04fce70 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/debug.go @@ -0,0 +1,159 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build debug + +package prefix + +import ( + "fmt" + "math" + "strings" +) + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func lenBase2(n uint) int { + return int(math.Ceil(math.Log2(float64(n + 1)))) +} +func padBase2(v, n uint, m int) string { + s := fmt.Sprintf("%b", 1< 0 { + return strings.Repeat(" ", pad) + s + } + return s +} + +func lenBase10(n int) int { + return int(math.Ceil(math.Log10(float64(n + 1)))) +} +func padBase10(n, m int) string { + s := fmt.Sprintf("%d", n) + if pad := m - len(s); pad > 0 { + return strings.Repeat(" ", pad) + s + } + return s +} + +func (rc RangeCodes) String() string { + var maxLen, maxBase int + for _, c := range rc { + maxLen = max(maxLen, int(c.Len)) + maxBase = max(maxBase, int(c.Base)) + } + + var ss []string + ss = append(ss, "{") + for i, c := range rc { + base := padBase10(int(c.Base), lenBase10(maxBase)) + if c.Len > 0 { + base += fmt.Sprintf("-%d", c.End()-1) + } + ss = append(ss, fmt.Sprintf("\t%s: {len: %s, range: %s},", + padBase10(int(i), lenBase10(len(rc)-1)), + padBase10(int(c.Len), lenBase10(maxLen)), + base, + )) + } + ss = append(ss, "}") + return strings.Join(ss, "\n") +} + +func (pc PrefixCodes) String() string { + var maxSym, maxLen, maxCnt int + for _, c := range pc { + maxSym = max(maxSym, int(c.Sym)) + maxLen = max(maxLen, int(c.Len)) + maxCnt = max(maxCnt, int(c.Cnt)) + } + + var ss []string + ss = append(ss, "{") + for _, c := range pc { + var cntStr string + if maxCnt > 0 { + cnt := int(32*float32(c.Cnt)/float32(maxCnt) + 0.5) + cntStr = fmt.Sprintf("%s |%s", + padBase10(int(c.Cnt), lenBase10(maxCnt)), + strings.Repeat("#", cnt), + ) + } + ss = append(ss, fmt.Sprintf("\t%s: %s, %s", + padBase10(int(c.Sym), lenBase10(maxSym)), + padBase2(uint(c.Val), uint(c.Len), maxLen), + cntStr, + )) + } + ss = append(ss, "}") + return strings.Join(ss, "\n") +} + +func (pd Decoder) String() string { + var ss []string + ss = append(ss, "{") + if len(pd.chunks) > 0 { + ss = append(ss, "\tchunks: {") + for i, c := range pd.chunks { + label := "sym" + if uint(c&countMask) > uint(pd.chunkBits) { + label = "idx" + } + ss = append(ss, fmt.Sprintf("\t\t%s: {%s: %s, len: %s}", + padBase2(uint(i), uint(pd.chunkBits), int(pd.chunkBits)), + label, padBase10(int(c>>countBits), 3), + padBase10(int(c&countMask), 2), + )) + } + ss = append(ss, "\t},") + + for j, links := range pd.links { + ss = append(ss, fmt.Sprintf("\tlinks[%d]: {", j)) + linkBits := lenBase2(uint(pd.linkMask)) + for i, c := range links { + ss = append(ss, fmt.Sprintf("\t\t%s: {sym: %s, len: %s},", + padBase2(uint(i), uint(linkBits), int(linkBits)), + padBase10(int(c>>countBits), 3), + padBase10(int(c&countMask), 2), + )) + } + ss = append(ss, "\t},") + } + } + ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pd.chunkMask)) + ss = append(ss, fmt.Sprintf("\tlinkMask: %b,", pd.linkMask)) + ss = append(ss, fmt.Sprintf("\tchunkBits: %d,", pd.chunkBits)) + ss = append(ss, fmt.Sprintf("\tMinBits: %d,", pd.MinBits)) + ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pd.NumSyms)) + ss = append(ss, "}") + return strings.Join(ss, "\n") +} + +func (pe Encoder) String() string { + var maxLen int + for _, c := range pe.chunks { + maxLen = max(maxLen, int(c&countMask)) + } + + var ss []string + ss = append(ss, "{") + if len(pe.chunks) > 0 { + ss = append(ss, "\tchunks: {") + for i, c := range pe.chunks { + ss = append(ss, fmt.Sprintf("\t\t%s: %s,", + padBase10(i, 3), + padBase2(uint(c>>countBits), uint(c&countMask), maxLen), + )) + } + ss = append(ss, "\t},") + } + ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pe.chunkMask)) + ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pe.NumSyms)) + ss = append(ss, "}") + return strings.Join(ss, "\n") +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/decoder.go b/vendor/github.com/dsnet/compress/internal/prefix/decoder.go new file mode 100644 index 0000000..a9bc2dc --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/decoder.go @@ -0,0 +1,136 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "sort" + + "github.com/dsnet/compress/internal" +) + +// The algorithm used to decode variable length codes is based on the lookup +// method in zlib. If the code is less-than-or-equal to maxChunkBits, +// then the symbol can be decoded using a single lookup into the chunks table. +// Otherwise, the links table will be used for a second level lookup. +// +// The chunks slice is keyed by the contents of the bit buffer ANDed with +// the chunkMask to avoid a out-of-bounds lookup. The value of chunks is a tuple +// that is decoded as follow: +// +// var length = chunks[bitBuffer&chunkMask] & countMask +// var symbol = chunks[bitBuffer&chunkMask] >> countBits +// +// If the decoded length is larger than chunkBits, then an overflow link table +// must be used for further decoding. In this case, the symbol is actually the +// index into the links tables. The second-level links table returned is +// processed in the same way as the chunks table. +// +// if length > chunkBits { +// var index = symbol // Previous symbol is index into links tables +// length = links[index][bitBuffer>>chunkBits & linkMask] & countMask +// symbol = links[index][bitBuffer>>chunkBits & linkMask] >> countBits +// } +// +// See the following: +// http://www.gzip.org/algorithm.txt + +type Decoder struct { + chunks []uint32 // First-level lookup map + links [][]uint32 // Second-level lookup map + chunkMask uint32 // Mask the length of the chunks table + linkMask uint32 // Mask the length of the link table + chunkBits uint32 // Bit-length of the chunks table + + MinBits uint32 // The minimum number of bits to safely make progress + NumSyms uint32 // Number of symbols +} + +// Init initializes Decoder according to the codes provided. +func (pd *Decoder) Init(codes PrefixCodes) { + // Handle special case trees. + if len(codes) <= 1 { + switch { + case len(codes) == 0: // Empty tree (should error if used later) + *pd = Decoder{chunks: pd.chunks[:0], links: pd.links[:0], NumSyms: 0} + case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero) + pd.chunks = append(pd.chunks[:0], codes[0].Sym< c.Len { + minBits = c.Len + } + if maxBits < c.Len { + maxBits = c.Len + } + } + + // Allocate chunks table as needed. + const maxChunkBits = 9 // This can be tuned for better performance + pd.NumSyms = uint32(len(codes)) + pd.MinBits = minBits + pd.chunkBits = maxBits + if pd.chunkBits > maxChunkBits { + pd.chunkBits = maxChunkBits + } + numChunks := 1 << pd.chunkBits + pd.chunks = allocUint32s(pd.chunks, numChunks) + pd.chunkMask = uint32(numChunks - 1) + + // Allocate links tables as needed. + pd.links = pd.links[:0] + pd.linkMask = 0 + if pd.chunkBits < maxBits { + numLinks := 1 << (maxBits - pd.chunkBits) + pd.linkMask = uint32(numLinks - 1) + + var linkIdx uint32 + for i := range pd.chunks { + pd.chunks[i] = 0 // Logic below relies on zero value as uninitialized + } + for _, c := range codes { + if c.Len > pd.chunkBits && pd.chunks[c.Val&pd.chunkMask] == 0 { + pd.chunks[c.Val&pd.chunkMask] = (linkIdx << countBits) | (pd.chunkBits + 1) + linkIdx++ + } + } + + pd.links = extendSliceUint32s(pd.links, int(linkIdx)) + linksFlat := allocUint32s(pd.links[0], numLinks*int(linkIdx)) + for i, j := 0, 0; i < len(pd.links); i, j = i+1, j+numLinks { + pd.links[i] = linksFlat[j : j+numLinks] + } + } + + // Fill out chunks and links tables with values. + for _, c := range codes { + chunk := c.Sym<> countBits + links := pd.links[linkIdx] + skip := 1 << uint(c.Len-pd.chunkBits) + for j := int(c.Val >> pd.chunkBits); j < len(links); j += skip { + links[j] = chunk + } + } + } +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/encoder.go b/vendor/github.com/dsnet/compress/internal/prefix/encoder.go new file mode 100644 index 0000000..4424a01 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/encoder.go @@ -0,0 +1,66 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "sort" + + "github.com/dsnet/compress/internal" +) + +type Encoder struct { + chunks []uint32 // First-level lookup map + chunkMask uint32 // Mask the length of the chunks table + + NumSyms uint32 // Number of symbols +} + +// Init initializes Encoder according to the codes provided. +func (pe *Encoder) Init(codes PrefixCodes) { + // Handle special case trees. + if len(codes) <= 1 { + switch { + case len(codes) == 0: // Empty tree (should error if used later) + *pe = Encoder{chunks: pe.chunks[:0], NumSyms: 0} + case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero) + pe.chunks = append(pe.chunks[:0], codes[0].Val< 0; n >>= 1 { + numChunks <<= 1 + } + pe.NumSyms = uint32(len(codes)) + +retry: + // Allocate and reset chunks. + pe.chunks = allocUint32s(pe.chunks, numChunks) + pe.chunkMask = uint32(numChunks - 1) + for i := range pe.chunks { + pe.chunks[i] = 0 // Logic below relies on zero value as uninitialized + } + + // Insert each symbol, checking that there are no conflicts. + for _, c := range codes { + if pe.chunks[c.Sym&pe.chunkMask] > 0 { + // Collision found our "hash" table, so grow and try again. + numChunks <<= 1 + goto retry + } + pe.chunks[c.Sym&pe.chunkMask] = c.Val<> uint(c.Len) + } + return sum == 0 || len(pc) == 0 +} + +// checkPrefixes reports whether all codes have non-overlapping prefixes. +func (pc PrefixCodes) checkPrefixes() bool { + for i, c1 := range pc { + for j, c2 := range pc { + mask := uint32(1)< 0 { + c.Val = internal.ReverseUint32N(c.Val, uint(c.Len)) + if vals[c.Len].Cnt > 0 && vals[c.Len].Val+1 != c.Val { + return false + } + vals[c.Len].Val = c.Val + vals[c.Len].Cnt++ + } + } + + // Rule 2. + var last PrefixCode + for _, v := range vals { + if v.Cnt > 0 { + curVal := v.Val - v.Cnt + 1 + if last.Cnt != 0 && last.Val >= curVal { + return false + } + last = v + } + } + return true +} + +// GenerateLengths assigns non-zero bit-lengths to all codes. Codes with high +// frequency counts will be assigned shorter codes to reduce bit entropy. +// This function is used primarily by compressors. +// +// The input codes must have the Cnt field populated, be sorted by count. +// Even if a code has a count of 0, a non-zero bit-length will be assigned. +// +// The result will have the Len field populated. The algorithm used guarantees +// that Len <= maxBits and that it is a complete prefix tree. The resulting +// codes will remain sorted by count. +func GenerateLengths(codes PrefixCodes, maxBits uint) error { + if len(codes) <= 1 { + if len(codes) == 1 { + codes[0].Len = 0 + } + return nil + } + + // Verify that the codes are in ascending order by count. + cntLast := codes[0].Cnt + for _, c := range codes[1:] { + if c.Cnt < cntLast { + return errorf(errors.Invalid, "non-monotonically increasing symbol counts") + } + cntLast = c.Cnt + } + + // Construct a Huffman tree used to generate the bit-lengths. + // + // The Huffman tree is a binary tree where each symbol lies as a leaf node + // on this tree. The length of the prefix code to assign is the depth of + // that leaf from the root. The Huffman algorithm, which runs in O(n), + // is used to generate the tree. It assumes that codes are sorted in + // increasing order of frequency. + // + // The algorithm is as follows: + // 1. Start with two queues, F and Q, where F contains all of the starting + // symbols sorted such that symbols with lowest counts come first. + // 2. While len(F)+len(Q) > 1: + // 2a. Dequeue the node from F or Q that has the lowest weight as N0. + // 2b. Dequeue the node from F or Q that has the lowest weight as N1. + // 2c. Create a new node N that has N0 and N1 as its children. + // 2d. Enqueue N into the back of Q. + // 3. The tree's root node is Q[0]. + type node struct { + cnt uint32 + + // n0 or c0 represent the left child of this node. + // Since Go does not have unions, only one of these will be set. + // Similarly, n1 or c1 represent the right child of this node. + // + // If n0 or n1 is set, then it represents a "pointer" to another + // node in the Huffman tree. Since Go's pointer analysis cannot reason + // that these node pointers do not escape (golang.org/issue/13493), + // we use an index to a node in the nodes slice as a pseudo-pointer. + // + // If c0 or c1 is set, then it represents a leaf "node" in the + // Huffman tree. The leaves are the PrefixCode values themselves. + n0, n1 int // Index to child nodes + c0, c1 *PrefixCode + } + var nodeIdx int + var nodeArr [1024]node // Large enough to handle most cases on the stack + nodes := nodeArr[:] + if len(nodes) < len(codes) { + nodes = make([]node, len(codes)) // Number of internal nodes < number of leaves + } + freqs, queue := codes, nodes[:0] + for len(freqs)+len(queue) > 1 { + // These are the two smallest nodes at the front of freqs and queue. + var n node + if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { + n.c0, freqs = &freqs[0], freqs[1:] + n.cnt += n.c0.Cnt + } else { + n.cnt += queue[0].cnt + n.n0 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0] + nodeIdx++ + queue = queue[1:] + } + if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { + n.c1, freqs = &freqs[0], freqs[1:] + n.cnt += n.c1.Cnt + } else { + n.cnt += queue[0].cnt + n.n1 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0] + nodeIdx++ + queue = queue[1:] + } + queue = append(queue, n) + } + rootIdx := nodeIdx + + // Search the whole binary tree, noting when we hit each leaf node. + // We do not care about the exact Huffman tree structure, but rather we only + // care about depth of each of the leaf nodes. That is, the depth determines + // how long each symbol is in bits. + // + // Since the number of leaves is n, there is at most n internal nodes. + // Thus, this algorithm runs in O(n). + var fixBits bool + var explore func(int, uint) + explore = func(rootIdx int, level uint) { + root := &nodes[rootIdx] + + // Explore left branch. + if root.c0 == nil { + explore(root.n0, level+1) + } else { + fixBits = fixBits || (level > maxBits) + root.c0.Len = uint32(level) + } + + // Explore right branch. + if root.c1 == nil { + explore(root.n1, level+1) + } else { + fixBits = fixBits || (level > maxBits) + root.c1.Len = uint32(level) + } + } + explore(rootIdx, 1) + + // Fix the bit-lengths if we violate the maxBits requirement. + if fixBits { + // Create histogram for number of symbols with each bit-length. + var symBitsArr [valueBits + 1]uint32 + symBits := symBitsArr[:] // symBits[nb] indicates number of symbols using nb bits + for _, c := range codes { + for int(c.Len) >= len(symBits) { + symBits = append(symBits, 0) + } + symBits[c.Len]++ + } + + // Fudge the tree such that the largest bit-length is <= maxBits. + // This is accomplish by effectively doing a tree rotation. That is, we + // increase the bit-length of some higher frequency code, so that the + // bit-lengths of lower frequency codes can be decreased. + // + // Visually, this looks like the following transform: + // + // Level Before After + // __ ___ + // / \ / \ + // n-1 X / \ /\ /\ + // n X /\ X X X X + // n+1 X X + // + var treeRotate func(uint) + treeRotate = func(nb uint) { + if symBits[nb-1] == 0 { + treeRotate(nb - 1) + } + symBits[nb-1] -= 1 // Push this node to the level below + symBits[nb] += 3 // This level gets one node from above, two from below + symBits[nb+1] -= 2 // Push two nodes to the level above + } + for i := uint(len(symBits)) - 1; i > maxBits; i-- { + for symBits[i] > 0 { + treeRotate(i - 1) + } + } + + // Assign bit-lengths to each code. Since codes is sorted in increasing + // order of frequency, that means that the most frequently used symbols + // should have the shortest bit-lengths. Thus, we copy symbols to codes + // from the back of codes first. + cs := codes + for nb, cnt := range symBits { + if cnt > 0 { + pos := len(cs) - int(cnt) + cs2 := cs[pos:] + for i := range cs2 { + cs2[i].Len = uint32(nb) + } + cs = cs[:pos] + } + } + if len(cs) != 0 { + panic("not all codes were used up") + } + } + + if internal.Debug && !codes.checkLengths() { + panic("incomplete prefix tree detected") + } + return nil +} + +// GeneratePrefixes assigns a prefix value to all codes according to the +// bit-lengths. This function is used by both compressors and decompressors. +// +// The input codes must have the Sym and Len fields populated and be +// sorted by symbol. The bit-lengths of each code must be properly allocated, +// such that it forms a complete tree. +// +// The result will have the Val field populated and will produce a canonical +// prefix tree. The resulting codes will remain sorted by symbol. +func GeneratePrefixes(codes PrefixCodes) error { + if len(codes) <= 1 { + if len(codes) == 1 { + if codes[0].Len != 0 { + return errorf(errors.Invalid, "degenerate prefix tree with one node") + } + codes[0].Val = 0 + } + return nil + } + + // Compute basic statistics on the symbols. + var bitCnts [valueBits + 1]uint + c0 := codes[0] + bitCnts[c0.Len]++ + minBits, maxBits, symLast := c0.Len, c0.Len, c0.Sym + for _, c := range codes[1:] { + if c.Sym <= symLast { + return errorf(errors.Invalid, "non-unique or non-monotonically increasing symbols") + } + if minBits > c.Len { + minBits = c.Len + } + if maxBits < c.Len { + maxBits = c.Len + } + bitCnts[c.Len]++ // Histogram of bit counts + symLast = c.Sym // Keep track of last symbol + } + if minBits == 0 { + return errorf(errors.Invalid, "invalid prefix bit-length") + } + + // Compute the next code for a symbol of a given bit length. + var nextCodes [valueBits + 1]uint + var code uint + for i := minBits; i <= maxBits; i++ { + code <<= 1 + nextCodes[i] = code + code += bitCnts[i] + } + if code != 1<= n { + return s[:n] + } + return make([]uint32, n, n*3/2) +} + +func extendSliceUint32s(s [][]uint32, n int) [][]uint32 { + if cap(s) >= n { + return s[:n] + } + ss := make([][]uint32, n, n*3/2) + copy(ss, s[:cap(s)]) + return ss +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/range.go b/vendor/github.com/dsnet/compress/internal/prefix/range.go new file mode 100644 index 0000000..b7eddad --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/range.go @@ -0,0 +1,93 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +type RangeCode struct { + Base uint32 // Starting base offset of the range + Len uint32 // Bit-length of a subsequent integer to add to base offset +} +type RangeCodes []RangeCode + +type RangeEncoder struct { + rcs RangeCodes + lut [1024]uint32 + minBase uint +} + +// End reports the non-inclusive ending range. +func (rc RangeCode) End() uint32 { return rc.Base + (1 << rc.Len) } + +// MakeRangeCodes creates a RangeCodes, where each region is assumed to be +// contiguously stacked, without any gaps, with bit-lengths taken from bits. +func MakeRangeCodes(minBase uint, bits []uint) (rc RangeCodes) { + for _, nb := range bits { + rc = append(rc, RangeCode{Base: uint32(minBase), Len: uint32(nb)}) + minBase += 1 << nb + } + return rc +} + +// Base reports the inclusive starting range for all ranges. +func (rcs RangeCodes) Base() uint32 { return rcs[0].Base } + +// End reports the non-inclusive ending range for all ranges. +func (rcs RangeCodes) End() uint32 { return rcs[len(rcs)-1].End() } + +// checkValid reports whether the RangeCodes is valid. In order to be valid, +// the following must hold true: +// rcs[i-1].Base <= rcs[i].Base +// rcs[i-1].End <= rcs[i].End +// rcs[i-1].End >= rcs[i].Base +// +// Practically speaking, each range must be increasing and must not have any +// gaps in between. It is okay for ranges to overlap. +func (rcs RangeCodes) checkValid() bool { + if len(rcs) == 0 { + return false + } + pre := rcs[0] + for _, cur := range rcs[1:] { + preBase, preEnd := pre.Base, pre.End() + curBase, curEnd := cur.Base, cur.End() + if preBase > curBase || preEnd > curEnd || preEnd < curBase { + return false + } + pre = cur + } + return true +} + +func (re *RangeEncoder) Init(rcs RangeCodes) { + if !rcs.checkValid() { + panic("invalid range codes") + } + *re = RangeEncoder{rcs: rcs, minBase: uint(rcs.Base())} + for sym, rc := range rcs { + base := int(rc.Base) - int(re.minBase) + end := int(rc.End()) - int(re.minBase) + if base >= len(re.lut) { + break + } + if end > len(re.lut) { + end = len(re.lut) + } + for i := base; i < end; i++ { + re.lut[i] = uint32(sym) + } + } +} + +func (re *RangeEncoder) Encode(offset uint) (sym uint) { + if idx := int(offset - re.minBase); idx < len(re.lut) { + return uint(re.lut[idx]) + } + sym = uint(re.lut[len(re.lut)-1]) +retry: + if int(sym) >= len(re.rcs) || re.rcs[sym].Base > uint32(offset) { + return sym - 1 + } + sym++ + goto retry // Avoid for-loop so that this function can be inlined +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/reader.go b/vendor/github.com/dsnet/compress/internal/prefix/reader.go new file mode 100644 index 0000000..e6252c9 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/reader.go @@ -0,0 +1,335 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "bufio" + "bytes" + "encoding/binary" + "io" + "strings" + + "github.com/dsnet/compress" + "github.com/dsnet/compress/internal" + "github.com/dsnet/compress/internal/errors" +) + +// Reader implements a prefix decoder. If the input io.Reader satisfies the +// compress.ByteReader or compress.BufferedReader interface, then it also +// guarantees that it will never read more bytes than is necessary. +// +// For high performance, provide an io.Reader that satisfies the +// compress.BufferedReader interface. If the input does not satisfy either +// compress.ByteReader or compress.BufferedReader, then it will be internally +// wrapped with a bufio.Reader. +type Reader struct { + Offset int64 // Number of bytes read from the underlying io.Reader + + rd io.Reader + byteRd compress.ByteReader // Set if rd is a ByteReader + bufRd compress.BufferedReader // Set if rd is a BufferedReader + + bufBits uint64 // Buffer to hold some bits + numBits uint // Number of valid bits in bufBits + bigEndian bool // Do we treat input bytes as big endian? + + // These fields are only used if rd is a compress.BufferedReader. + bufPeek []byte // Buffer for the Peek data + discardBits int // Number of bits to discard from reader + fedBits uint // Number of bits fed in last call to PullBits + + // These fields are used to reduce allocations. + bb *buffer + br *bytesReader + sr *stringReader + bu *bufio.Reader +} + +// Init initializes the bit Reader to read from r. If bigEndian is true, then +// bits will be read starting from the most-significant bits of a byte +// (as done in bzip2), otherwise it will read starting from the +// least-significant bits of a byte (such as for deflate and brotli). +func (pr *Reader) Init(r io.Reader, bigEndian bool) { + *pr = Reader{ + rd: r, + bigEndian: bigEndian, + + bb: pr.bb, + br: pr.br, + sr: pr.sr, + bu: pr.bu, + } + switch rr := r.(type) { + case *bytes.Buffer: + if pr.bb == nil { + pr.bb = new(buffer) + } + *pr.bb = buffer{Buffer: rr} + pr.bufRd = pr.bb + case *bytes.Reader: + if pr.br == nil { + pr.br = new(bytesReader) + } + *pr.br = bytesReader{Reader: rr} + pr.bufRd = pr.br + case *strings.Reader: + if pr.sr == nil { + pr.sr = new(stringReader) + } + *pr.sr = stringReader{Reader: rr} + pr.bufRd = pr.sr + case compress.BufferedReader: + pr.bufRd = rr + case compress.ByteReader: + pr.byteRd = rr + default: + if pr.bu == nil { + pr.bu = bufio.NewReader(nil) + } + pr.bu.Reset(r) + pr.rd, pr.bufRd = pr.bu, pr.bu + } +} + +// BitsRead reports the total number of bits emitted from any Read method. +func (pr *Reader) BitsRead() int64 { + offset := 8*pr.Offset - int64(pr.numBits) + if pr.bufRd != nil { + discardBits := pr.discardBits + int(pr.fedBits-pr.numBits) + offset = 8*pr.Offset + int64(discardBits) + } + return offset +} + +// IsBufferedReader reports whether the underlying io.Reader is also a +// compress.BufferedReader. +func (pr *Reader) IsBufferedReader() bool { + return pr.bufRd != nil +} + +// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment. +func (pr *Reader) ReadPads() uint { + nb := pr.numBits % 8 + val := uint(pr.bufBits & uint64(1<>= nb + pr.numBits -= nb + return val +} + +// Read reads bytes into buf. +// The bit-ordering mode does not affect this method. +func (pr *Reader) Read(buf []byte) (cnt int, err error) { + if pr.numBits > 0 { + if pr.numBits%8 != 0 { + return 0, errorf(errors.Invalid, "non-aligned bit buffer") + } + for cnt = 0; len(buf) > cnt && pr.numBits > 0; cnt++ { + if pr.bigEndian { + buf[cnt] = internal.ReverseLUT[byte(pr.bufBits)] + } else { + buf[cnt] = byte(pr.bufBits) + } + pr.bufBits >>= 8 + pr.numBits -= 8 + } + return cnt, nil + } + if _, err := pr.Flush(); err != nil { + return 0, err + } + cnt, err = pr.rd.Read(buf) + pr.Offset += int64(cnt) + return cnt, err +} + +// ReadOffset reads an offset value using the provided RangeCodes indexed by +// the symbol read. +func (pr *Reader) ReadOffset(pd *Decoder, rcs RangeCodes) uint { + rc := rcs[pr.ReadSymbol(pd)] + return uint(rc.Base) + pr.ReadBits(uint(rc.Len)) +} + +// TryReadBits attempts to read nb bits using the contents of the bit buffer +// alone. It returns the value and whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pr *Reader) TryReadBits(nb uint) (uint, bool) { + if pr.numBits < nb { + return 0, false + } + val := uint(pr.bufBits & uint64(1<>= nb + pr.numBits -= nb + return val, true +} + +// ReadBits reads nb bits in from the underlying reader. +func (pr *Reader) ReadBits(nb uint) uint { + if err := pr.PullBits(nb); err != nil { + errors.Panic(err) + } + val := uint(pr.bufBits & uint64(1<>= nb + pr.numBits -= nb + return val +} + +// TryReadSymbol attempts to decode the next symbol using the contents of the +// bit buffer alone. It returns the decoded symbol and whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pr *Reader) TryReadSymbol(pd *Decoder) (uint, bool) { + if pr.numBits < uint(pd.MinBits) || len(pd.chunks) == 0 { + return 0, false + } + chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] + nb := uint(chunk & countMask) + if nb > pr.numBits || nb > uint(pd.chunkBits) { + return 0, false + } + pr.bufBits >>= nb + pr.numBits -= nb + return uint(chunk >> countBits), true +} + +// ReadSymbol reads the next symbol using the provided prefix Decoder. +func (pr *Reader) ReadSymbol(pd *Decoder) uint { + if len(pd.chunks) == 0 { + panicf(errors.Invalid, "decode with empty prefix tree") + } + + nb := uint(pd.MinBits) + for { + if err := pr.PullBits(nb); err != nil { + errors.Panic(err) + } + chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] + nb = uint(chunk & countMask) + if nb > uint(pd.chunkBits) { + linkIdx := chunk >> countBits + chunk = pd.links[linkIdx][uint32(pr.bufBits>>pd.chunkBits)&pd.linkMask] + nb = uint(chunk & countMask) + } + if nb <= pr.numBits { + pr.bufBits >>= nb + pr.numBits -= nb + return uint(chunk >> countBits) + } + } +} + +// Flush updates the read offset of the underlying ByteReader. +// If reader is a compress.BufferedReader, then this calls Discard to update +// the read offset. +func (pr *Reader) Flush() (int64, error) { + if pr.bufRd == nil { + return pr.Offset, nil + } + + // Update the number of total bits to discard. + pr.discardBits += int(pr.fedBits - pr.numBits) + pr.fedBits = pr.numBits + + // Discard some bytes to update read offset. + var err error + nd := (pr.discardBits + 7) / 8 // Round up to nearest byte + nd, err = pr.bufRd.Discard(nd) + pr.discardBits -= nd * 8 // -7..0 + pr.Offset += int64(nd) + + // These are invalid after Discard. + pr.bufPeek = nil + return pr.Offset, err +} + +// PullBits ensures that at least nb bits exist in the bit buffer. +// If the underlying reader is a compress.BufferedReader, then this will fill +// the bit buffer with as many bits as possible, relying on Peek and Discard to +// properly advance the read offset. Otherwise, it will use ReadByte to fill the +// buffer with just the right number of bits. +func (pr *Reader) PullBits(nb uint) error { + if pr.bufRd != nil { + pr.discardBits += int(pr.fedBits - pr.numBits) + for { + if len(pr.bufPeek) == 0 { + pr.fedBits = pr.numBits // Don't discard bits just added + if _, err := pr.Flush(); err != nil { + return err + } + + // Peek no more bytes than necessary. + // The computation for cntPeek computes the minimum number of + // bytes to Peek to fill nb bits. + var err error + cntPeek := int(nb+(-nb&7)) / 8 + if cntPeek < pr.bufRd.Buffered() { + cntPeek = pr.bufRd.Buffered() + } + pr.bufPeek, err = pr.bufRd.Peek(cntPeek) + pr.bufPeek = pr.bufPeek[int(pr.numBits/8):] // Skip buffered bits + if len(pr.bufPeek) == 0 { + if pr.numBits >= nb { + break + } + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + } + + n := int(64-pr.numBits) / 8 // Number of bytes to copy to bit buffer + if len(pr.bufPeek) >= 8 { + // Starting with Go 1.7, the compiler should use a wide integer + // load here if the architecture supports it. + u := binary.LittleEndian.Uint64(pr.bufPeek) + if pr.bigEndian { + // Swap all the bits within each byte. + u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 + u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 + u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 + } + + pr.bufBits |= u << pr.numBits + pr.numBits += uint(n * 8) + pr.bufPeek = pr.bufPeek[n:] + break + } else { + if n > len(pr.bufPeek) { + n = len(pr.bufPeek) + } + for _, c := range pr.bufPeek[:n] { + if pr.bigEndian { + c = internal.ReverseLUT[c] + } + pr.bufBits |= uint64(c) << pr.numBits + pr.numBits += 8 + } + pr.bufPeek = pr.bufPeek[n:] + if pr.numBits > 56 { + break + } + } + } + pr.fedBits = pr.numBits + } else { + for pr.numBits < nb { + c, err := pr.byteRd.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + if pr.bigEndian { + c = internal.ReverseLUT[c] + } + pr.bufBits |= uint64(c) << pr.numBits + pr.numBits += 8 + pr.Offset++ + } + } + return nil +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/wrap.go b/vendor/github.com/dsnet/compress/internal/prefix/wrap.go new file mode 100644 index 0000000..49906d4 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/wrap.go @@ -0,0 +1,146 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "bytes" + "io" + "strings" +) + +// For some of the common Readers, we wrap and extend them to satisfy the +// compress.BufferedReader interface to improve performance. + +type buffer struct { + *bytes.Buffer +} + +type bytesReader struct { + *bytes.Reader + pos int64 + buf []byte + arr [512]byte +} + +type stringReader struct { + *strings.Reader + pos int64 + buf []byte + arr [512]byte +} + +func (r *buffer) Buffered() int { + return r.Len() +} + +func (r *buffer) Peek(n int) ([]byte, error) { + b := r.Bytes() + if len(b) < n { + return b, io.EOF + } + return b[:n], nil +} + +func (r *buffer) Discard(n int) (int, error) { + b := r.Next(n) + if len(b) < n { + return len(b), io.EOF + } + return n, nil +} + +func (r *bytesReader) Buffered() int { + r.update() + if r.Len() > len(r.buf) { + return len(r.buf) + } + return r.Len() +} + +func (r *bytesReader) Peek(n int) ([]byte, error) { + if n > len(r.arr) { + return nil, io.ErrShortBuffer + } + + // Return sub-slice of local buffer if possible. + r.update() + if len(r.buf) >= n { + return r.buf[:n], nil + } + + // Fill entire local buffer, and return appropriate sub-slice. + cnt, err := r.ReadAt(r.arr[:], r.pos) + r.buf = r.arr[:cnt] + if cnt < n { + return r.arr[:cnt], err + } + return r.arr[:n], nil +} + +func (r *bytesReader) Discard(n int) (int, error) { + var err error + if n > r.Len() { + n, err = r.Len(), io.EOF + } + r.Seek(int64(n), io.SeekCurrent) + return n, err +} + +// update reslices the internal buffer to be consistent with the read offset. +func (r *bytesReader) update() { + pos, _ := r.Seek(0, io.SeekCurrent) + if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { + r.buf, r.pos = r.buf[off:], pos + } else { + r.buf, r.pos = nil, pos + } +} + +func (r *stringReader) Buffered() int { + r.update() + if r.Len() > len(r.buf) { + return len(r.buf) + } + return r.Len() +} + +func (r *stringReader) Peek(n int) ([]byte, error) { + if n > len(r.arr) { + return nil, io.ErrShortBuffer + } + + // Return sub-slice of local buffer if possible. + r.update() + if len(r.buf) >= n { + return r.buf[:n], nil + } + + // Fill entire local buffer, and return appropriate sub-slice. + cnt, err := r.ReadAt(r.arr[:], r.pos) + r.buf = r.arr[:cnt] + if cnt < n { + return r.arr[:cnt], err + } + return r.arr[:n], nil +} + +func (r *stringReader) Discard(n int) (int, error) { + var err error + if n > r.Len() { + n, err = r.Len(), io.EOF + } + r.Seek(int64(n), io.SeekCurrent) + return n, err +} + +// update reslices the internal buffer to be consistent with the read offset. +func (r *stringReader) update() { + pos, _ := r.Seek(0, io.SeekCurrent) + if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { + r.buf, r.pos = r.buf[off:], pos + } else { + r.buf, r.pos = nil, pos + } +} diff --git a/vendor/github.com/dsnet/compress/internal/prefix/writer.go b/vendor/github.com/dsnet/compress/internal/prefix/writer.go new file mode 100644 index 0000000..c978390 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/prefix/writer.go @@ -0,0 +1,166 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +package prefix + +import ( + "encoding/binary" + "io" + + "github.com/dsnet/compress/internal/errors" +) + +// Writer implements a prefix encoder. For performance reasons, Writer will not +// write bytes immediately to the underlying stream. +type Writer struct { + Offset int64 // Number of bytes written to the underlying io.Writer + + wr io.Writer + bufBits uint64 // Buffer to hold some bits + numBits uint // Number of valid bits in bufBits + bigEndian bool // Are bits written in big-endian order? + + buf [512]byte + cntBuf int +} + +// Init initializes the bit Writer to write to w. If bigEndian is true, then +// bits will be written starting from the most-significant bits of a byte +// (as done in bzip2), otherwise it will write starting from the +// least-significant bits of a byte (such as for deflate and brotli). +func (pw *Writer) Init(w io.Writer, bigEndian bool) { + *pw = Writer{wr: w, bigEndian: bigEndian} + return +} + +// BitsWritten reports the total number of bits issued to any Write method. +func (pw *Writer) BitsWritten() int64 { + return 8*pw.Offset + 8*int64(pw.cntBuf) + int64(pw.numBits) +} + +// WritePads writes 0-7 bits to the bit buffer to achieve byte-alignment. +func (pw *Writer) WritePads(v uint) { + nb := -pw.numBits & 7 + pw.bufBits |= uint64(v) << pw.numBits + pw.numBits += nb +} + +// Write writes bytes from buf. +// The bit-ordering mode does not affect this method. +func (pw *Writer) Write(buf []byte) (cnt int, err error) { + if pw.numBits > 0 || pw.cntBuf > 0 { + if pw.numBits%8 != 0 { + return 0, errorf(errors.Invalid, "non-aligned bit buffer") + } + if _, err := pw.Flush(); err != nil { + return 0, err + } + } + cnt, err = pw.wr.Write(buf) + pw.Offset += int64(cnt) + return cnt, err +} + +// WriteOffset writes ofs in a (sym, extra) fashion using the provided prefix +// Encoder and RangeEncoder. +func (pw *Writer) WriteOffset(ofs uint, pe *Encoder, re *RangeEncoder) { + sym := re.Encode(ofs) + pw.WriteSymbol(sym, pe) + rc := re.rcs[sym] + pw.WriteBits(ofs-uint(rc.Base), uint(rc.Len)) +} + +// TryWriteBits attempts to write nb bits using the contents of the bit buffer +// alone. It reports whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pw *Writer) TryWriteBits(v, nb uint) bool { + if 64-pw.numBits < nb { + return false + } + pw.bufBits |= uint64(v) << pw.numBits + pw.numBits += nb + return true +} + +// WriteBits writes nb bits of v to the underlying writer. +func (pw *Writer) WriteBits(v, nb uint) { + if _, err := pw.PushBits(); err != nil { + errors.Panic(err) + } + pw.bufBits |= uint64(v) << pw.numBits + pw.numBits += nb +} + +// TryWriteSymbol attempts to encode the next symbol using the contents of the +// bit buffer alone. It reports whether it succeeded. +// +// This method is designed to be inlined for performance reasons. +func (pw *Writer) TryWriteSymbol(sym uint, pe *Encoder) bool { + chunk := pe.chunks[uint32(sym)&pe.chunkMask] + nb := uint(chunk & countMask) + if 64-pw.numBits < nb { + return false + } + pw.bufBits |= uint64(chunk>>countBits) << pw.numBits + pw.numBits += nb + return true +} + +// WriteSymbol writes the symbol using the provided prefix Encoder. +func (pw *Writer) WriteSymbol(sym uint, pe *Encoder) { + if _, err := pw.PushBits(); err != nil { + errors.Panic(err) + } + chunk := pe.chunks[uint32(sym)&pe.chunkMask] + nb := uint(chunk & countMask) + pw.bufBits |= uint64(chunk>>countBits) << pw.numBits + pw.numBits += nb +} + +// Flush flushes all complete bytes from the bit buffer to the byte buffer, and +// then flushes all bytes in the byte buffer to the underlying writer. +// After this call, the bit Writer is will only withhold 7 bits at most. +func (pw *Writer) Flush() (int64, error) { + if pw.numBits < 8 && pw.cntBuf == 0 { + return pw.Offset, nil + } + if _, err := pw.PushBits(); err != nil { + return pw.Offset, err + } + cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) + pw.cntBuf -= cnt + pw.Offset += int64(cnt) + return pw.Offset, err +} + +// PushBits pushes as many bytes as possible from the bit buffer to the byte +// buffer, reporting the number of bits pushed. +func (pw *Writer) PushBits() (uint, error) { + if pw.cntBuf >= len(pw.buf)-8 { + cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) + pw.cntBuf -= cnt + pw.Offset += int64(cnt) + if err != nil { + return 0, err + } + } + + u := pw.bufBits + if pw.bigEndian { + // Swap all the bits within each byte. + u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 + u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 + u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 + } + // Starting with Go 1.7, the compiler should use a wide integer + // store here if the architecture supports it. + binary.LittleEndian.PutUint64(pw.buf[pw.cntBuf:], u) + + nb := pw.numBits / 8 // Number of bytes to copy from bit buffer + pw.cntBuf += int(nb) + pw.bufBits >>= 8 * nb + pw.numBits -= 8 * nb + return 8 * nb, nil +} diff --git a/vendor/github.com/dsnet/compress/internal/release.go b/vendor/github.com/dsnet/compress/internal/release.go new file mode 100644 index 0000000..0990be1 --- /dev/null +++ b/vendor/github.com/dsnet/compress/internal/release.go @@ -0,0 +1,21 @@ +// Copyright 2015, Joe Tsai. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.md file. + +// +build !debug,!gofuzz + +package internal + +// Debug indicates whether the debug build tag was set. +// +// If set, programs may choose to print with more human-readable +// debug information and also perform sanity checks that would otherwise be too +// expensive to run in a release build. +const Debug = false + +// GoFuzz indicates whether the gofuzz build tag was set. +// +// If set, programs may choose to disable certain checks (like checksums) that +// would be nearly impossible for gofuzz to properly get right. +// If GoFuzz is set, it implies that Debug is set as well. +const GoFuzz = false diff --git a/vendor/github.com/dsnet/compress/zbench.sh b/vendor/github.com/dsnet/compress/zbench.sh new file mode 100755 index 0000000..0205920 --- /dev/null +++ b/vendor/github.com/dsnet/compress/zbench.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +# zbench wraps internal/tool/bench and is useful for comparing benchmarks from +# the implementations in this repository relative to other implementations. +# +# See internal/tool/bench/main.go for more details. +cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/bench +go run $(go list -f '{{ join .GoFiles "\n" }}') "$@" diff --git a/vendor/github.com/dsnet/compress/zfuzz.sh b/vendor/github.com/dsnet/compress/zfuzz.sh new file mode 100755 index 0000000..42958ed --- /dev/null +++ b/vendor/github.com/dsnet/compress/zfuzz.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +# zfuzz wraps internal/tool/fuzz and is useful for fuzz testing each of +# the implementations in this repository. +cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/fuzz +./fuzz.sh "$@" diff --git a/vendor/github.com/dsnet/compress/zprof.sh b/vendor/github.com/dsnet/compress/zprof.sh new file mode 100755 index 0000000..3cd535b --- /dev/null +++ b/vendor/github.com/dsnet/compress/zprof.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +if [ $# == 0 ]; then + echo "Usage: $0 PKG_PATH TEST_ARGS..." + echo "" + echo "Runs coverage and performance benchmarks for a given package." + echo "The results are stored in the _zprof_ directory." + echo "" + echo "Example:" + echo " $0 flate -test.bench=Decode/Twain/Default" + exit 1 +fi + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_PATH=$1 +PKG_NAME=$(basename $PKG_PATH) +shift + +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR $PKG_PATH/$PKG_NAME.test" SIGINT SIGTERM EXIT + +( + cd $DIR/$PKG_PATH + + # Print the go version. + go version + + # Perform coverage profiling. + go test github.com/dsnet/compress/$PKG_PATH -coverprofile $TMPDIR/cover.profile + if [ $? != 0 ]; then exit 1; fi + go tool cover -html $TMPDIR/cover.profile -o cover.html + + # Perform performance profiling. + if [ $# != 0 ]; then + go test -c github.com/dsnet/compress/$PKG_PATH + if [ $? != 0 ]; then exit 1; fi + ./$PKG_NAME.test -test.cpuprofile $TMPDIR/cpu.profile -test.memprofile $TMPDIR/mem.profile -test.run - "$@" + PPROF="go tool pprof" + $PPROF -output=cpu.svg -web $PKG_NAME.test $TMPDIR/cpu.profile 2> /dev/null + $PPROF -output=cpu.html -weblist=. $PKG_NAME.test $TMPDIR/cpu.profile 2> /dev/null + $PPROF -output=mem_objects.svg -alloc_objects -web $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + $PPROF -output=mem_objects.html -alloc_objects -weblist=. $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + $PPROF -output=mem_space.svg -alloc_space -web $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + $PPROF -output=mem_space.html -alloc_space -weblist=. $PKG_NAME.test $TMPDIR/mem.profile 2> /dev/null + fi + + rm -rf $DIR/_zprof_/$PKG_NAME + mkdir -p $DIR/_zprof_/$PKG_NAME + mv *.html *.svg $DIR/_zprof_/$PKG_NAME 2> /dev/null +) diff --git a/vendor/github.com/dsnet/compress/ztest.sh b/vendor/github.com/dsnet/compress/ztest.sh new file mode 100755 index 0000000..d4c1993 --- /dev/null +++ b/vendor/github.com/dsnet/compress/ztest.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright 2017, Joe Tsai. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE.md file. + +cd $(go list -f '{{ .Dir }}' github.com/dsnet/compress) + +BOLD="\x1b[1mRunning: " +PASS="\x1b[32mPASS" +FAIL="\x1b[31mFAIL" +RESET="\x1b[0m" + +echo -e "${BOLD}fmt${RESET}" +RET_FMT=$(find . -name "*.go" | egrep -v "/(_.*_|\..*|testdata)/" | xargs gofmt -d) +if [[ ! -z "$RET_FMT" ]]; then echo "$RET_FMT"; echo; fi + +echo -e "${BOLD}test${RESET}" +RET_TEST=$(go test -race ./... | egrep -v "^(ok|[?])\s+") +if [[ ! -z "$RET_TEST" ]]; then echo "$RET_TEST"; echo; fi + +echo -e "${BOLD}staticcheck${RESET}" +RET_SCHK=$(staticcheck \ + -ignore " + github.com/dsnet/compress/internal/prefix/*.go:SA4016 + github.com/dsnet/compress/brotli/*.go:SA4016 + " ./... 2>&1) +if [[ ! -z "$RET_SCHK" ]]; then echo "$RET_SCHK"; echo; fi + +echo -e "${BOLD}vet${RESET}" +RET_VET=$(go vet ./... 2>&1 | + egrep -v "^flate/dict_decoder.go:(.*)WriteByte" | + egrep -v "^exit status") +if [[ ! -z "$RET_VET" ]]; then echo "$RET_VET"; echo; fi + +echo -e "${BOLD}lint${RESET}" +RET_LINT=$(golint ./... 2>&1 | + egrep -v "should have comment(.*)or be unexported" | + egrep -v "^(.*)type name will be used as(.*)by other packages" | + egrep -v "^brotli/transform.go:(.*)replace i [+]= 1 with i[+]{2}" | + egrep -v "^internal/prefix/prefix.go:(.*)replace symBits(.*) [-]= 1 with symBits(.*)[-]{2}" | + egrep -v "^xflate/common.go:(.*)NoCompression should be of the form" | + egrep -v "^exit status") +if [[ ! -z "$RET_LINT" ]]; then echo "$RET_LINT"; echo; fi + +if [[ ! -z "$RET_FMT" ]] || [ ! -z "$RET_TEST" ] || [[ ! -z "$RET_VET" ]] || [[ ! -z "$RET_SCHK" ]] || [[ ! -z "$RET_LINT" ]] || [[ ! -z "$RET_SPELL" ]]; then + echo -e "${FAIL}${RESET}"; exit 1 +else + echo -e "${PASS}${RESET}"; exit 0 +fi diff --git a/vendor/github.com/mholt/archiver/.gitignore b/vendor/github.com/mholt/archiver/.gitignore new file mode 100644 index 0000000..ac8f8b2 --- /dev/null +++ b/vendor/github.com/mholt/archiver/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +_gitignore +builds/ +*.test +cmd/archiver/archiver diff --git a/vendor/github.com/mholt/archiver/.travis.yml b/vendor/github.com/mholt/archiver/.travis.yml new file mode 100644 index 0000000..9d5f79a --- /dev/null +++ b/vendor/github.com/mholt/archiver/.travis.yml @@ -0,0 +1,21 @@ +language: go + +go: + - 1.7.1 + +env: + - CGO_ENABLED=0 + +install: + - go get -t ./... + - go get github.com/golang/lint/golint + - go get github.com/gordonklaus/ineffassign + +script: + - diff <(echo -n) <(gofmt -s -d .) + - ineffassign . + - go vet ./... + - go test ./... + +after_script: + - golint ./... diff --git a/vendor/github.com/mholt/archiver/LICENSE b/vendor/github.com/mholt/archiver/LICENSE new file mode 100644 index 0000000..315d04f --- /dev/null +++ b/vendor/github.com/mholt/archiver/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Matthew Holt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/mholt/archiver/README.md b/vendor/github.com/mholt/archiver/README.md new file mode 100644 index 0000000..47eeb54 --- /dev/null +++ b/vendor/github.com/mholt/archiver/README.md @@ -0,0 +1,81 @@ +archiver [![archiver GoDoc](https://img.shields.io/badge/reference-godoc-blue.svg?style=flat-square)](https://godoc.org/github.com/mholt/archiver) [![Linux Build Status](https://img.shields.io/travis/mholt/archiver.svg?style=flat-square&label=linux+build)](https://travis-ci.org/mholt/archiver) [![Windows Build Status](https://img.shields.io/appveyor/ci/mholt/archiver.svg?style=flat-square&label=windows+build)](https://ci.appveyor.com/project/mholt/archiver) +======== + +Package archiver makes it trivially easy to make and extract common archive formats such as .zip, and .tar.gz. Simply name the input and output file(s). + +Files are put into the root of the archive; directories are recursively added, preserving structure. + +The `archiver` command runs the same cross-platform and has no external dependencies (not even libc); powered by the Go standard library, [dsnet/compress](https://github.com/dsnet/compress), [nwaples/rardecode](https://github.com/nwaples/rardecode), and [ulikunitz/xz](https://github.com/ulikunitz/xz). Enjoy! + +Supported formats/extensions: + +- .zip +- .tar +- .tar.gz & .tgz +- .tar.bz2 & .tbz2 +- .tar.xz & .txz +- .rar (open only) + + +## Install + +```bash +go get github.com/mholt/archiver/cmd/archiver +``` + +Or download binaries from the [releases](https://github.com/mholt/archiver/releases) page. + + +## Command Use + +Make a new archive: + +```bash +$ archiver make [archive name] [input files...] +``` + +(At least one input file is required.) + +To extract an archive: + +```bash +$ archiver open [archive name] [destination] +``` + +(The destination path is optional; default is current directory.) + +The archive name must end with a supported file extension—this is how it knows what kind of archive to make. Run `archiver -h` for more help. + + +## Library Use + +```go +import "github.com/mholt/archiver" +``` + +Create a .zip file: + +```go +err := archiver.Zip.Make("output.zip", []string{"file.txt", "folder"}) +``` + +Extract a .zip file: + +```go +err := archiver.Zip.Open("input.zip", "output_folder") +``` + +Working with other file formats is exactly the same, but with [their own Archiver implementations](https://godoc.org/github.com/mholt/archiver#Archiver). + + + +## FAQ + +#### Can I list a file in one folder to go into a different folder in the archive? + +No. This works just like your OS would make an archive in the file explorer: organize your input files to mirror the structure you want in the archive. + + +#### Can it add files to an existing archive? + +Nope. This is a simple tool; it just makes new archives or extracts existing ones. diff --git a/vendor/github.com/mholt/archiver/appveyor.yml b/vendor/github.com/mholt/archiver/appveyor.yml new file mode 100644 index 0000000..f8f804f --- /dev/null +++ b/vendor/github.com/mholt/archiver/appveyor.yml @@ -0,0 +1,32 @@ +version: "{build}" + +os: Windows Server 2012 R2 + +clone_folder: c:\gopath\src\github.com\mholt\archiver + +environment: + GOPATH: c:\gopath + CGO_ENABLED: 0 + +install: + - rmdir c:\go /s /q + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.1.windows-amd64.zip + - 7z x go1.7.1.windows-amd64.zip -y -oC:\ > NUL + - go version + - go env + - go get -t ./... + - go get github.com/golang/lint/golint + - go get github.com/gordonklaus/ineffassign + - set PATH=%GOPATH%\bin;%PATH% + +build: off + +test_script: + - go vet ./... + - go test ./... + - ineffassign . + +after_test: + - golint ./... + +deploy: off diff --git a/vendor/github.com/mholt/archiver/archiver.go b/vendor/github.com/mholt/archiver/archiver.go new file mode 100644 index 0000000..2561e0f --- /dev/null +++ b/vendor/github.com/mholt/archiver/archiver.go @@ -0,0 +1,78 @@ +package archiver + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + "runtime" +) + +// Archiver represent a archive format +type Archiver interface { + // Match checks supported files + Match(filename string) bool + // Make makes an archive. + Make(destination string, sources []string) error + // Open extracts an archive. + Open(source, destination string) error +} + +// SupportedFormats contains all supported archive formats +var SupportedFormats = map[string]Archiver{} + +// RegisterFormat adds a supported archive format +func RegisterFormat(name string, format Archiver) { + if _, ok := SupportedFormats[name]; ok { + log.Printf("Format %s already exists, skip!\n", name) + return + } + SupportedFormats[name] = format +} + +func writeNewFile(fpath string, in io.Reader, fm os.FileMode) error { + err := os.MkdirAll(filepath.Dir(fpath), 0755) + if err != nil { + return fmt.Errorf("%s: making directory for file: %v", fpath, err) + } + + out, err := os.Create(fpath) + if err != nil { + return fmt.Errorf("%s: creating new file: %v", fpath, err) + } + defer out.Close() + + err = out.Chmod(fm) + if err != nil && runtime.GOOS != "windows" { + return fmt.Errorf("%s: changing file mode: %v", fpath, err) + } + + _, err = io.Copy(out, in) + if err != nil { + return fmt.Errorf("%s: writing file: %v", fpath, err) + } + return nil +} + +func writeNewSymbolicLink(fpath string, target string) error { + err := os.MkdirAll(filepath.Dir(fpath), 0755) + if err != nil { + return fmt.Errorf("%s: making directory for file: %v", fpath, err) + } + + err = os.Symlink(target, fpath) + if err != nil { + return fmt.Errorf("%s: making symbolic link for: %v", fpath, err) + } + + return nil +} + +func mkdir(dirPath string) error { + err := os.MkdirAll(dirPath, 0755) + if err != nil { + return fmt.Errorf("%s: making directory: %v", dirPath, err) + } + return nil +} diff --git a/vendor/github.com/mholt/archiver/build.bash b/vendor/github.com/mholt/archiver/build.bash new file mode 100755 index 0000000..b1bef11 --- /dev/null +++ b/vendor/github.com/mholt/archiver/build.bash @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -ex + +# This script builds archiver for most common platforms. + +export CGO_ENABLED=0 + +cd cmd/archiver +GOOS=linux GOARCH=386 go build -o ../../builds/archiver_linux_386 +GOOS=linux GOARCH=amd64 go build -o ../../builds/archiver_linux_amd64 +GOOS=linux GOARCH=arm go build -o ../../builds/archiver_linux_arm7 +GOOS=linux GOARCH=arm64 go build -o ../../builds/archiver_linux_arm64 +GOOS=darwin GOARCH=amd64 go build -o ../../builds/archiver_mac_amd64 +GOOS=windows GOARCH=386 go build -o ../../builds/archiver_windows_386.exe +GOOS=windows GOARCH=amd64 go build -o ../../builds/archiver_windows_amd64.exe +GOOS=freebsd GOARCH=386 go build -o ../../builds/archiver_freebsd_386 +GOOS=freebsd GOARCH=amd64 go build -o ../../builds/archiver_freebsd_amd64 +GOOS=freebsd GOARCH=arm go build -o ../../builds/archiver_freebsd_arm7 +GOOS=openbsd GOARCH=386 go build -o ../../builds/archiver_openbsd_386 +GOOS=openbsd GOARCH=amd64 go build -o ../../builds/archiver_openbsd_amd64 +cd ../.. \ No newline at end of file diff --git a/vendor/github.com/mholt/archiver/rar.go b/vendor/github.com/mholt/archiver/rar.go new file mode 100644 index 0000000..a15a202 --- /dev/null +++ b/vendor/github.com/mholt/archiver/rar.go @@ -0,0 +1,78 @@ +package archiver + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/nwaples/rardecode" +) + +// Rar is for RAR archive format +var Rar rarFormat + +func init() { + RegisterFormat("Rar", Rar) +} + +type rarFormat struct{} + +func (rarFormat) Match(filename string) bool { + // TODO: read file header to identify the format + return strings.HasSuffix(strings.ToLower(filename), ".rar") +} + +// Make makes a .rar archive, but this is not implemented because +// RAR is a proprietary format. It is here only for symmetry with +// the other archive formats in this package. +func (rarFormat) Make(rarPath string, filePaths []string) error { + return fmt.Errorf("make %s: RAR not implemented (proprietary format)", rarPath) +} + +// Open extracts the RAR file at source and puts the contents +// into destination. +func (rarFormat) Open(source, destination string) error { + f, err := os.Open(source) + if err != nil { + return fmt.Errorf("%s: failed to open archive: %v", source, err) + } + defer f.Close() + + rr, err := rardecode.NewReader(f, "") + if err != nil { + return fmt.Errorf("%s: failed to create reader: %v", source, err) + } + + for { + header, err := rr.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + if header.IsDir { + err = mkdir(filepath.Join(destination, header.Name)) + if err != nil { + return err + } + continue + } + + // if files come before their containing folders, then we must + // create their folders before writing the file + err = mkdir(filepath.Dir(filepath.Join(destination, header.Name))) + if err != nil { + return err + } + + err = writeNewFile(filepath.Join(destination, header.Name), rr, header.Mode()) + if err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/mholt/archiver/tar.go b/vendor/github.com/mholt/archiver/tar.go new file mode 100644 index 0000000..c3231d4 --- /dev/null +++ b/vendor/github.com/mholt/archiver/tar.go @@ -0,0 +1,157 @@ +package archiver + +import ( + "archive/tar" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// Tar is for Tar format +var Tar tarFormat + +func init() { + RegisterFormat("Tar", Tar) +} + +type tarFormat struct{} + +func (tarFormat) Match(filename string) bool { + // TODO: read file header to identify the format + return strings.HasSuffix(strings.ToLower(filename), ".tar") +} + +// Make creates a .tar file at tarPath containing the +// contents of files listed in filePaths. File paths can +// be those of regular files or directories. Regular +// files are stored at the 'root' of the archive, and +// directories are recursively added. +func (tarFormat) Make(tarPath string, filePaths []string) error { + out, err := os.Create(tarPath) + if err != nil { + return fmt.Errorf("error creating %s: %v", tarPath, err) + } + defer out.Close() + + tarWriter := tar.NewWriter(out) + defer tarWriter.Close() + + return tarball(filePaths, tarWriter, tarPath) +} + +// tarball writes all files listed in filePaths into tarWriter, which is +// writing into a file located at dest. +func tarball(filePaths []string, tarWriter *tar.Writer, dest string) error { + for _, fpath := range filePaths { + err := tarFile(tarWriter, fpath, dest) + if err != nil { + return err + } + } + return nil +} + +// tarFile writes the file at source into tarWriter. It does so +// recursively for directories. +func tarFile(tarWriter *tar.Writer, source, dest string) error { + sourceInfo, err := os.Stat(source) + if err != nil { + return fmt.Errorf("%s: stat: %v", source, err) + } + + var baseDir string + if sourceInfo.IsDir() { + baseDir = filepath.Base(source) + } + + return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error walking to %s: %v", path, err) + } + + header, err := tar.FileInfoHeader(info, path) + if err != nil { + return fmt.Errorf("%s: making header: %v", path, err) + } + + if baseDir != "" { + header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) + } + + if header.Name == dest { + // our new tar file is inside the directory being archived; skip it + return nil + } + + if info.IsDir() { + header.Name += "/" + } + + err = tarWriter.WriteHeader(header) + if err != nil { + return fmt.Errorf("%s: writing header: %v", path, err) + } + + if info.IsDir() { + return nil + } + + if header.Typeflag == tar.TypeReg { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("%s: open: %v", path, err) + } + defer file.Close() + + _, err = io.CopyN(tarWriter, file, info.Size()) + if err != nil && err != io.EOF { + return fmt.Errorf("%s: copying contents: %v", path, err) + } + } + return nil + }) +} + +// Open untars source and puts the contents into destination. +func (tarFormat) Open(source, destination string) error { + f, err := os.Open(source) + if err != nil { + return fmt.Errorf("%s: failed to open archive: %v", source, err) + } + defer f.Close() + + return untar(tar.NewReader(f), destination) +} + +// untar un-tarballs the contents of tr into destination. +func untar(tr *tar.Reader, destination string) error { + for { + header, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + if err := untarFile(tr, header, destination); err != nil { + return err + } + } + return nil +} + +// untarFile untars a single file from tr with header header into destination. +func untarFile(tr *tar.Reader, header *tar.Header, destination string) error { + switch header.Typeflag { + case tar.TypeDir: + return mkdir(filepath.Join(destination, header.Name)) + case tar.TypeReg, tar.TypeRegA: + return writeNewFile(filepath.Join(destination, header.Name), tr, header.FileInfo().Mode()) + case tar.TypeSymlink: + return writeNewSymbolicLink(filepath.Join(destination, header.Name), header.Linkname) + default: + return fmt.Errorf("%s: unknown type flag: %c", header.Name, header.Typeflag) + } +} diff --git a/vendor/github.com/mholt/archiver/tarbz2.go b/vendor/github.com/mholt/archiver/tarbz2.go new file mode 100644 index 0000000..83e7fa5 --- /dev/null +++ b/vendor/github.com/mholt/archiver/tarbz2.go @@ -0,0 +1,66 @@ +package archiver + +import ( + "archive/tar" + "fmt" + "os" + "strings" + + "github.com/dsnet/compress/bzip2" +) + +// TarBz2 is for TarBz2 format +var TarBz2 tarBz2Format + +func init() { + RegisterFormat("TarBz2", TarBz2) +} + +type tarBz2Format struct{} + +func (tarBz2Format) Match(filename string) bool { + // TODO: read file header to identify the format + return strings.HasSuffix(strings.ToLower(filename), ".tar.bz2") || + strings.HasSuffix(strings.ToLower(filename), ".tbz2") +} + +// Make creates a .tar.bz2 file at tarbz2Path containing +// the contents of files listed in filePaths. File paths +// can be those of regular files or directories. Regular +// files are stored at the 'root' of the archive, and +// directories are recursively added. +func (tarBz2Format) Make(tarbz2Path string, filePaths []string) error { + out, err := os.Create(tarbz2Path) + if err != nil { + return fmt.Errorf("error creating %s: %v", tarbz2Path, err) + } + defer out.Close() + + bz2Writer, err := bzip2.NewWriter(out, nil) + if err != nil { + return fmt.Errorf("error compressing %s: %v", tarbz2Path, err) + } + defer bz2Writer.Close() + + tarWriter := tar.NewWriter(bz2Writer) + defer tarWriter.Close() + + return tarball(filePaths, tarWriter, tarbz2Path) +} + +// Open untars source and decompresses the contents into destination. +func (tarBz2Format) Open(source, destination string) error { + f, err := os.Open(source) + if err != nil { + return fmt.Errorf("%s: failed to open archive: %v", source, err) + } + defer f.Close() + + bz2r, err := bzip2.NewReader(f, nil) + if err != nil { + return fmt.Errorf("error decompressing %s: %v", source, err) + } + defer bz2r.Close() + + return untar(tar.NewReader(bz2r), destination) +} diff --git a/vendor/github.com/mholt/archiver/targz.go b/vendor/github.com/mholt/archiver/targz.go new file mode 100644 index 0000000..895b099 --- /dev/null +++ b/vendor/github.com/mholt/archiver/targz.go @@ -0,0 +1,60 @@ +package archiver + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "os" + "strings" +) + +// TarGz is for TarGz format +var TarGz tarGzFormat + +func init() { + RegisterFormat("TarGz", TarGz) +} + +type tarGzFormat struct{} + +func (tarGzFormat) Match(filename string) bool { + // TODO: read file header to identify the format + return strings.HasSuffix(strings.ToLower(filename), ".tar.gz") || + strings.HasSuffix(strings.ToLower(filename), ".tgz") +} + +// Make creates a .tar.gz file at targzPath containing +// the contents of files listed in filePaths. It works +// the same way Tar does, but with gzip compression. +func (tarGzFormat) Make(targzPath string, filePaths []string) error { + out, err := os.Create(targzPath) + if err != nil { + return fmt.Errorf("error creating %s: %v", targzPath, err) + } + defer out.Close() + + gzWriter := gzip.NewWriter(out) + defer gzWriter.Close() + + tarWriter := tar.NewWriter(gzWriter) + defer tarWriter.Close() + + return tarball(filePaths, tarWriter, targzPath) +} + +// Open untars source and decompresses the contents into destination. +func (tarGzFormat) Open(source, destination string) error { + f, err := os.Open(source) + if err != nil { + return fmt.Errorf("%s: failed to open archive: %v", source, err) + } + defer f.Close() + + gzr, err := gzip.NewReader(f) + if err != nil { + return fmt.Errorf("%s: create new gzip reader: %v", source, err) + } + defer gzr.Close() + + return untar(tar.NewReader(gzr), destination) +} diff --git a/vendor/github.com/mholt/archiver/tarxz.go b/vendor/github.com/mholt/archiver/tarxz.go new file mode 100644 index 0000000..9bffce7 --- /dev/null +++ b/vendor/github.com/mholt/archiver/tarxz.go @@ -0,0 +1,66 @@ +package archiver + +import ( + "archive/tar" + "fmt" + "os" + "strings" + + "github.com/ulikunitz/xz" +) + +// TarXZ is for TarXZ format +var TarXZ xzFormat + +func init() { + RegisterFormat("TarXZ", TarXZ) +} + +type xzFormat struct{} + +// Match returns whether filename matches this format. +func (xzFormat) Match(filename string) bool { + // TODO: read file header to identify the format + return strings.HasSuffix(strings.ToLower(filename), ".tar.xz") || + strings.HasSuffix(strings.ToLower(filename), ".txz") +} + +// Make creates a .tar.xz file at xzPath containing +// the contents of files listed in filePaths. File +// paths can be those of regular files or directories. +// Regular files are stored at the 'root' of the +// archive, and directories are recursively added. +func (xzFormat) Make(xzPath string, filePaths []string) error { + out, err := os.Create(xzPath) + if err != nil { + return fmt.Errorf("error creating %s: %v", xzPath, err) + } + defer out.Close() + + xzWriter, err := xz.NewWriter(out) + if err != nil { + return fmt.Errorf("error compressing %s: %v", xzPath, err) + } + defer xzWriter.Close() + + tarWriter := tar.NewWriter(xzWriter) + defer tarWriter.Close() + + return tarball(filePaths, tarWriter, xzPath) +} + +// Open untars source and decompresses the contents into destination. +func (xzFormat) Open(source, destination string) error { + f, err := os.Open(source) + if err != nil { + return fmt.Errorf("%s: failed to open archive: %v", source, err) + } + defer f.Close() + + xzReader, err := xz.NewReader(f) + if err != nil { + return fmt.Errorf("error decompressing %s: %v", source, err) + } + + return untar(tar.NewReader(xzReader), destination) +} diff --git a/vendor/github.com/mholt/archiver/zip.go b/vendor/github.com/mholt/archiver/zip.go new file mode 100644 index 0000000..9b53537 --- /dev/null +++ b/vendor/github.com/mholt/archiver/zip.go @@ -0,0 +1,180 @@ +// Package archiver makes it super easy to create and open .zip, +// .tar.gz, and .tar.bz2 files. +package archiver + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" +) + +// Zip is for Zip format +var Zip zipFormat + +func init() { + RegisterFormat("Zip", Zip) +} + +type zipFormat struct{} + +func (zipFormat) Match(filename string) bool { + // TODO: read file header to identify the format + return strings.HasSuffix(strings.ToLower(filename), ".zip") +} + +// Make creates a .zip file in the location zipPath containing +// the contents of files listed in filePaths. File paths +// can be those of regular files or directories. Regular +// files are stored at the 'root' of the archive, and +// directories are recursively added. +// +// Files with an extension for formats that are already +// compressed will be stored only, not compressed. +func (zipFormat) Make(zipPath string, filePaths []string) error { + out, err := os.Create(zipPath) + if err != nil { + return fmt.Errorf("error creating %s: %v", zipPath, err) + } + defer out.Close() + + w := zip.NewWriter(out) + for _, fpath := range filePaths { + err = zipFile(w, fpath) + if err != nil { + w.Close() + return err + } + } + + return w.Close() +} + +func zipFile(w *zip.Writer, source string) error { + sourceInfo, err := os.Stat(source) + if err != nil { + return fmt.Errorf("%s: stat: %v", source, err) + } + + var baseDir string + if sourceInfo.IsDir() { + baseDir = filepath.Base(source) + } + + return filepath.Walk(source, func(fpath string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("walking to %s: %v", fpath, err) + } + + header, err := zip.FileInfoHeader(info) + if err != nil { + return fmt.Errorf("%s: getting header: %v", fpath, err) + } + + if baseDir != "" { + header.Name = path.Join(baseDir, strings.TrimPrefix(fpath, source)) + } + + if info.IsDir() { + header.Name += "/" + header.Method = zip.Store + } else { + ext := strings.ToLower(path.Ext(header.Name)) + if _, ok := compressedFormats[ext]; ok { + header.Method = zip.Store + } else { + header.Method = zip.Deflate + } + } + + writer, err := w.CreateHeader(header) + if err != nil { + return fmt.Errorf("%s: making header: %v", fpath, err) + } + + if info.IsDir() { + return nil + } + + if header.Mode().IsRegular() { + file, err := os.Open(fpath) + if err != nil { + return fmt.Errorf("%s: opening: %v", fpath, err) + } + defer file.Close() + + _, err = io.CopyN(writer, file, info.Size()) + if err != nil && err != io.EOF { + return fmt.Errorf("%s: copying contents: %v", fpath, err) + } + } + + return nil + }) +} + +// Open unzips the .zip file at source into destination. +func (zipFormat) Open(source, destination string) error { + r, err := zip.OpenReader(source) + if err != nil { + return err + } + defer r.Close() + + for _, zf := range r.File { + if err := unzipFile(zf, destination); err != nil { + return err + } + } + + return nil +} + +func unzipFile(zf *zip.File, destination string) error { + if strings.HasSuffix(zf.Name, "/") { + return mkdir(filepath.Join(destination, zf.Name)) + } + + rc, err := zf.Open() + if err != nil { + return fmt.Errorf("%s: open compressed file: %v", zf.Name, err) + } + defer rc.Close() + + return writeNewFile(filepath.Join(destination, zf.Name), rc, zf.FileInfo().Mode()) +} + +// compressedFormats is a (non-exhaustive) set of lowercased +// file extensions for formats that are typically already +// compressed. Compressing already-compressed files often +// results in a larger file, so when possible, we check this +// set to avoid that. +var compressedFormats = map[string]struct{}{ + ".7z": {}, + ".avi": {}, + ".bz2": {}, + ".cab": {}, + ".gif": {}, + ".gz": {}, + ".jar": {}, + ".jpeg": {}, + ".jpg": {}, + ".lz": {}, + ".lzma": {}, + ".mov": {}, + ".mp3": {}, + ".mp4": {}, + ".mpeg": {}, + ".mpg": {}, + ".png": {}, + ".rar": {}, + ".tbz2": {}, + ".tgz": {}, + ".txz": {}, + ".xz": {}, + ".zip": {}, + ".zipx": {}, +} diff --git a/vendor/github.com/nwaples/rardecode/LICENSE b/vendor/github.com/nwaples/rardecode/LICENSE new file mode 100644 index 0000000..0050f92 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2015, Nicholas Waples +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/nwaples/rardecode/README.md b/vendor/github.com/nwaples/rardecode/README.md new file mode 100644 index 0000000..513464c --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/README.md @@ -0,0 +1,4 @@ +# rardecode +[![GoDoc](https://godoc.org/github.com/nwaples/rardecode?status.svg)](https://godoc.org/github.com/nwaples/rardecode) + +A go package for reading RAR archives. diff --git a/vendor/github.com/nwaples/rardecode/archive.go b/vendor/github.com/nwaples/rardecode/archive.go new file mode 100644 index 0000000..8929f12 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/archive.go @@ -0,0 +1,306 @@ +package rardecode + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +const ( + maxSfxSize = 0x100000 // maximum number of bytes to read when searching for RAR signature + sigPrefix = "Rar!\x1A\x07" + + fileFmt15 = iota + 1 // Version 1.5 archive file format + fileFmt50 // Version 5.0 archive file format +) + +var ( + errNoSig = errors.New("rardecode: RAR signature not found") + errVerMismatch = errors.New("rardecode: volume version mistmatch") + errCorruptHeader = errors.New("rardecode: corrupt block header") + errCorruptFileHeader = errors.New("rardecode: corrupt file header") + errBadHeaderCrc = errors.New("rardecode: bad header crc") + errUnknownArc = errors.New("rardecode: unknown archive version") + errUnknownDecoder = errors.New("rardecode: unknown decoder version") + errUnsupportedDecoder = errors.New("rardecode: unsupported decoder version") + errArchiveContinues = errors.New("rardecode: archive continues in next volume") + errArchiveEnd = errors.New("rardecode: archive end reached") + errDecoderOutOfData = errors.New("rardecode: decoder expected more data than is in packed file") + + reDigits = regexp.MustCompile(`\d+`) +) + +type readBuf []byte + +func (b *readBuf) byte() byte { + v := (*b)[0] + *b = (*b)[1:] + return v +} + +func (b *readBuf) uint16() uint16 { + v := uint16((*b)[0]) | uint16((*b)[1])<<8 + *b = (*b)[2:] + return v +} + +func (b *readBuf) uint32() uint32 { + v := uint32((*b)[0]) | uint32((*b)[1])<<8 | uint32((*b)[2])<<16 | uint32((*b)[3])<<24 + *b = (*b)[4:] + return v +} + +func (b *readBuf) bytes(n int) []byte { + v := (*b)[:n] + *b = (*b)[n:] + return v +} + +func (b *readBuf) uvarint() uint64 { + var x uint64 + var s uint + for i, n := range *b { + if n < 0x80 { + *b = (*b)[i+1:] + return x | uint64(n)< '9' || v.file[i+3] < '0' || v.file[i+3] > '9') { + v.file = v.file[:i+2] + "00" + return + } + } + // new style volume naming + if !v.old { + // find all numbers in volume name + m := reDigits.FindAllStringIndex(v.file, -1) + if l := len(m); l > 1 { + // More than 1 match so assume name.part###of###.rar style. + // Take the last 2 matches where the first is the volume number. + m = m[l-2 : l] + if strings.Contains(v.file[m[0][1]:m[1][0]], ".") || !strings.Contains(v.file[:m[0][0]], ".") { + // Didn't match above style as volume had '.' between the two numbers or didnt have a '.' + // before the first match. Use the second number as volume number. + m = m[1:] + } + } + // extract and increment volume number + lo, hi := m[0][0], m[0][1] + n, err := strconv.Atoi(v.file[lo:hi]) + if err != nil { + n = 0 + } else { + n++ + } + // volume number must use at least the same number of characters as previous volume + vol := fmt.Sprintf("%0"+fmt.Sprint(hi-lo)+"d", n) + v.file = v.file[:lo] + vol + v.file[hi:] + return + } + // old style volume naming + i := strings.LastIndex(v.file, ".") + // get file extension + b := []byte(v.file[i+1:]) + // start incrementing volume number digits from rightmost + for j := 2; j >= 0; j-- { + if b[j] != '9' { + b[j]++ + break + } + // digit overflow + if j == 0 { + // last character before '.' + b[j] = 'A' + } else { + // set to '0' and loop to next character + b[j] = '0' + } + } + v.file = v.file[:i+1] + string(b) +} + +func (v *volume) next() (*fileBlockHeader, error) { + for { + var atEOF bool + + h, err := v.fileBlockReader.next() + switch err { + case errArchiveContinues: + case io.EOF: + // Read all of volume without finding an end block. The only way + // to tell if the archive continues is to try to open the next volume. + atEOF = true + default: + return h, err + } + + v.f.Close() + v.nextVolName() + v.f, err = os.Open(v.dir + v.file) // Open next volume file + if err != nil { + if atEOF && os.IsNotExist(err) { + // volume not found so assume that the archive has ended + return nil, io.EOF + } + return nil, err + } + v.num++ + v.br.Reset(v.f) + ver, err := findSig(v.br) + if err != nil { + return nil, err + } + if v.version() != ver { + return nil, errVerMismatch + } + v.reset() // reset encryption + } +} + +func (v *volume) Close() error { + // may be nil if os.Open fails in next() + if v.f == nil { + return nil + } + return v.f.Close() +} + +func openVolume(name, password string) (*volume, error) { + var err error + v := new(volume) + v.dir, v.file = filepath.Split(name) + v.f, err = os.Open(name) + if err != nil { + return nil, err + } + v.br = bufio.NewReader(v.f) + v.fileBlockReader, err = newFileBlockReader(v.br, password) + if err != nil { + v.f.Close() + return nil, err + } + return v, nil +} + +func newFileBlockReader(br *bufio.Reader, pass string) (fileBlockReader, error) { + runes := []rune(pass) + if len(runes) > maxPassword { + pass = string(runes[:maxPassword]) + } + ver, err := findSig(br) + if err != nil { + return nil, err + } + switch ver { + case fileFmt15: + return newArchive15(br, pass), nil + case fileFmt50: + return newArchive50(br, pass), nil + } + return nil, errUnknownArc +} diff --git a/vendor/github.com/nwaples/rardecode/archive15.go b/vendor/github.com/nwaples/rardecode/archive15.go new file mode 100644 index 0000000..260176c --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/archive15.go @@ -0,0 +1,468 @@ +package rardecode + +import ( + "bufio" + "bytes" + "crypto/sha1" + "errors" + "hash" + "hash/crc32" + "io" + "io/ioutil" + "strconv" + "strings" + "time" + "unicode/utf16" +) + +const ( + // block types + blockArc = 0x73 + blockFile = 0x74 + blockService = 0x7a + blockEnd = 0x7b + + // block flags + blockHasData = 0x8000 + + // archive block flags + arcVolume = 0x0001 + arcSolid = 0x0008 + arcNewNaming = 0x0010 + arcEncrypted = 0x0080 + + // file block flags + fileSplitBefore = 0x0001 + fileSplitAfter = 0x0002 + fileEncrypted = 0x0004 + fileSolid = 0x0010 + fileWindowMask = 0x00e0 + fileLargeData = 0x0100 + fileUnicode = 0x0200 + fileSalt = 0x0400 + fileVersion = 0x0800 + fileExtTime = 0x1000 + + // end block flags + endArcNotLast = 0x0001 + + saltSize = 8 // size of salt for calculating AES keys + cacheSize30 = 4 // number of AES keys to cache + hashRounds = 0x40000 +) + +var ( + errMultipleDecoders = errors.New("rardecode: multiple decoders in a single archive not supported") +) + +type blockHeader15 struct { + htype byte // block header type + flags uint16 + data readBuf // header data + dataSize int64 // size of extra block data +} + +// fileHash32 implements fileChecksum for 32-bit hashes +type fileHash32 struct { + hash.Hash32 // hash to write file contents to + sum uint32 // 32bit checksum for file +} + +func (h *fileHash32) valid() bool { + return h.sum == h.Sum32() +} + +// archive15 implements fileBlockReader for RAR 1.5 file format archives +type archive15 struct { + byteReader // reader for current block data + v *bufio.Reader // reader for current archive volume + dec decoder // current decoder + decVer byte // current decoder version + multi bool // archive is multi-volume + old bool // archive uses old naming scheme + solid bool // archive is a solid archive + encrypted bool + pass []uint16 // password in UTF-16 + checksum fileHash32 // file checksum + buf readBuf // temporary buffer + keyCache [cacheSize30]struct { // cache of previously calculated decryption keys + salt []byte + key []byte + iv []byte + } +} + +// Calculates the key and iv for AES decryption given a password and salt. +func calcAes30Params(pass []uint16, salt []byte) (key, iv []byte) { + p := make([]byte, 0, len(pass)*2+len(salt)) + for _, v := range pass { + p = append(p, byte(v), byte(v>>8)) + } + p = append(p, salt...) + + hash := sha1.New() + iv = make([]byte, 16) + s := make([]byte, 0, hash.Size()) + for i := 0; i < hashRounds; i++ { + hash.Write(p) + hash.Write([]byte{byte(i), byte(i >> 8), byte(i >> 16)}) + if i%(hashRounds/16) == 0 { + s = hash.Sum(s[:0]) + iv[i/(hashRounds/16)] = s[4*4+3] + } + } + key = hash.Sum(s[:0]) + key = key[:16] + + for k := key; len(k) >= 4; k = k[4:] { + k[0], k[1], k[2], k[3] = k[3], k[2], k[1], k[0] + } + return key, iv +} + +// parseDosTime converts a 32bit DOS time value to time.Time +func parseDosTime(t uint32) time.Time { + n := int(t) + sec := n & 0x1f << 1 + min := n >> 5 & 0x3f + hr := n >> 11 & 0x1f + day := n >> 16 & 0x1f + mon := time.Month(n >> 21 & 0x0f) + yr := n>>25&0x7f + 1980 + return time.Date(yr, mon, day, hr, min, sec, 0, time.Local) +} + +// decodeName decodes a non-unicode filename from a file header. +func decodeName(buf []byte) string { + i := bytes.IndexByte(buf, 0) + if i < 0 { + return string(buf) // filename is UTF-8 + } + + name := buf[:i] + encName := readBuf(buf[i+1:]) + if len(encName) < 2 { + return "" // invalid encoding + } + highByte := uint16(encName.byte()) << 8 + flags := encName.byte() + flagBits := 8 + var wchars []uint16 // decoded characters are UTF-16 + for len(wchars) < len(name) && len(encName) > 0 { + if flagBits == 0 { + flags = encName.byte() + flagBits = 8 + if len(encName) == 0 { + break + } + } + switch flags >> 6 { + case 0: + wchars = append(wchars, uint16(encName.byte())) + case 1: + wchars = append(wchars, uint16(encName.byte())|highByte) + case 2: + if len(encName) < 2 { + break + } + wchars = append(wchars, encName.uint16()) + case 3: + n := encName.byte() + b := name[len(wchars):] + if l := int(n&0x7f) + 2; l < len(b) { + b = b[:l] + } + if n&0x80 > 0 { + if len(encName) < 1 { + break + } + ec := encName.byte() + for _, c := range b { + wchars = append(wchars, uint16(c+ec)|highByte) + } + } else { + for _, c := range b { + wchars = append(wchars, uint16(c)) + } + } + } + flags <<= 2 + flagBits -= 2 + } + return string(utf16.Decode(wchars)) +} + +// readExtTimes reads and parses the optional extra time field from the file header. +func readExtTimes(f *fileBlockHeader, b *readBuf) { + if len(*b) < 2 { + return // invalid, not enough data + } + flags := b.uint16() + + ts := []*time.Time{&f.ModificationTime, &f.CreationTime, &f.AccessTime} + + for i, t := range ts { + n := flags >> uint((3-i)*4) + if n&0x8 == 0 { + continue + } + if i != 0 { // ModificationTime already read so skip + if len(*b) < 4 { + return // invalid, not enough data + } + *t = parseDosTime(b.uint32()) + } + if n&0x4 > 0 { + *t = t.Add(time.Second) + } + n &= 0x3 + if n == 0 { + continue + } + if len(*b) < int(n) { + return // invalid, not enough data + } + // add extra time data in 100's of nanoseconds + d := time.Duration(0) + for j := 3 - n; j < n; j++ { + d |= time.Duration(b.byte()) << (j * 8) + } + d *= 100 + *t = t.Add(d) + } +} + +func (a *archive15) getKeys(salt []byte) (key, iv []byte) { + // check cache of keys + for _, v := range a.keyCache { + if bytes.Equal(v.salt[:], salt) { + return v.key, v.iv + } + } + key, iv = calcAes30Params(a.pass, salt) + + // save a copy in the cache + copy(a.keyCache[1:], a.keyCache[:]) + a.keyCache[0].salt = append([]byte(nil), salt...) // copy so byte slice can be reused + a.keyCache[0].key = key + a.keyCache[0].iv = iv + + return key, iv +} + +func (a *archive15) parseFileHeader(h *blockHeader15) (*fileBlockHeader, error) { + f := new(fileBlockHeader) + + f.first = h.flags&fileSplitBefore == 0 + f.last = h.flags&fileSplitAfter == 0 + + f.solid = h.flags&fileSolid > 0 + f.IsDir = h.flags&fileWindowMask == fileWindowMask + if !f.IsDir { + f.winSize = uint(h.flags&fileWindowMask)>>5 + 16 + } + + b := h.data + if len(b) < 21 { + return nil, errCorruptFileHeader + } + + f.PackedSize = h.dataSize + f.UnPackedSize = int64(b.uint32()) + f.HostOS = b.byte() + 1 + if f.HostOS > HostOSBeOS { + f.HostOS = HostOSUnknown + } + a.checksum.sum = b.uint32() + + f.ModificationTime = parseDosTime(b.uint32()) + unpackver := b.byte() // decoder version + method := b.byte() - 0x30 // decryption method + namesize := int(b.uint16()) + f.Attributes = int64(b.uint32()) + if h.flags&fileLargeData > 0 { + if len(b) < 8 { + return nil, errCorruptFileHeader + } + _ = b.uint32() // already read large PackedSize in readBlockHeader + f.UnPackedSize |= int64(b.uint32()) << 32 + f.UnKnownSize = f.UnPackedSize == -1 + } else if int32(f.UnPackedSize) == -1 { + f.UnKnownSize = true + f.UnPackedSize = -1 + } + if len(b) < namesize { + return nil, errCorruptFileHeader + } + name := b.bytes(namesize) + if h.flags&fileUnicode == 0 { + f.Name = string(name) + } else { + f.Name = decodeName(name) + } + // Rar 4.x uses '\' as file separator + f.Name = strings.Replace(f.Name, "\\", "/", -1) + + if h.flags&fileVersion > 0 { + // file version is stored as ';n' appended to file name + i := strings.LastIndex(f.Name, ";") + if i > 0 { + j, err := strconv.Atoi(f.Name[i+1:]) + if err == nil && j >= 0 { + f.Version = j + f.Name = f.Name[:i] + } + } + } + + var salt []byte + if h.flags&fileSalt > 0 { + if len(b) < saltSize { + return nil, errCorruptFileHeader + } + salt = b.bytes(saltSize) + } + if h.flags&fileExtTime > 0 { + readExtTimes(f, &b) + } + + if !f.first { + return f, nil + } + // fields only needed for first block in a file + if h.flags&fileEncrypted > 0 && len(salt) == saltSize { + f.key, f.iv = a.getKeys(salt) + } + a.checksum.Reset() + f.cksum = &a.checksum + if method == 0 { + return f, nil + } + if a.dec == nil { + switch unpackver { + case 15, 20, 26: + return nil, errUnsupportedDecoder + case 29: + a.dec = new(decoder29) + default: + return nil, errUnknownDecoder + } + a.decVer = unpackver + } else if a.decVer != unpackver { + return nil, errMultipleDecoders + } + f.decoder = a.dec + return f, nil +} + +// readBlockHeader returns the next block header in the archive. +// It will return io.EOF if there were no bytes read. +func (a *archive15) readBlockHeader() (*blockHeader15, error) { + var err error + b := a.buf[:7] + r := io.Reader(a.v) + if a.encrypted { + salt := a.buf[:saltSize] + _, err = io.ReadFull(r, salt) + if err != nil { + return nil, err + } + key, iv := a.getKeys(salt) + r = newAesDecryptReader(r, key, iv) + err = readFull(r, b) + } else { + _, err = io.ReadFull(r, b) + } + if err != nil { + return nil, err + } + + crc := b.uint16() + hash := crc32.NewIEEE() + hash.Write(b) + h := new(blockHeader15) + h.htype = b.byte() + h.flags = b.uint16() + size := b.uint16() + if size < 7 { + return nil, errCorruptHeader + } + size -= 7 + if int(size) > cap(a.buf) { + a.buf = readBuf(make([]byte, size)) + } + h.data = a.buf[:size] + if err := readFull(r, h.data); err != nil { + return nil, err + } + hash.Write(h.data) + if crc != uint16(hash.Sum32()) { + return nil, errBadHeaderCrc + } + if h.flags&blockHasData > 0 { + if len(h.data) < 4 { + return nil, errCorruptHeader + } + h.dataSize = int64(h.data.uint32()) + } + if (h.htype == blockService || h.htype == blockFile) && h.flags&fileLargeData > 0 { + if len(h.data) < 25 { + return nil, errCorruptHeader + } + b := h.data[21:25] + h.dataSize |= int64(b.uint32()) << 32 + } + return h, nil +} + +// next advances to the next file block in the archive +func (a *archive15) next() (*fileBlockHeader, error) { + for { + // could return an io.EOF here as 1.5 archives may not have an end block. + h, err := a.readBlockHeader() + if err != nil { + return nil, err + } + a.byteReader = limitByteReader(a.v, h.dataSize) // reader for block data + + switch h.htype { + case blockFile: + return a.parseFileHeader(h) + case blockArc: + a.encrypted = h.flags&arcEncrypted > 0 + a.multi = h.flags&arcVolume > 0 + a.old = h.flags&arcNewNaming == 0 + a.solid = h.flags&arcSolid > 0 + case blockEnd: + if h.flags&endArcNotLast == 0 || !a.multi { + return nil, errArchiveEnd + } + return nil, errArchiveContinues + default: + _, err = io.Copy(ioutil.Discard, a.byteReader) + } + if err != nil { + return nil, err + } + } +} + +func (a *archive15) version() int { return fileFmt15 } + +func (a *archive15) reset() { + a.encrypted = false // reset encryption when opening new volume file +} + +func (a *archive15) isSolid() bool { + return a.solid +} + +// newArchive15 creates a new fileBlockReader for a Version 1.5 archive +func newArchive15(r *bufio.Reader, password string) fileBlockReader { + a := new(archive15) + a.v = r + a.pass = utf16.Encode([]rune(password)) // convert to UTF-16 + a.checksum.Hash32 = crc32.NewIEEE() + a.buf = readBuf(make([]byte, 100)) + return a +} diff --git a/vendor/github.com/nwaples/rardecode/archive50.go b/vendor/github.com/nwaples/rardecode/archive50.go new file mode 100644 index 0000000..1d8f850 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/archive50.go @@ -0,0 +1,475 @@ +package rardecode + +import ( + "bufio" + "bytes" + "crypto/hmac" + "crypto/sha256" + "errors" + "hash" + "hash/crc32" + "io" + "io/ioutil" + "time" +) + +const ( + // block types + block5Arc = 1 + block5File = 2 + block5Service = 3 + block5Encrypt = 4 + block5End = 5 + + // block flags + block5HasExtra = 0x0001 + block5HasData = 0x0002 + block5DataNotFirst = 0x0008 + block5DataNotLast = 0x0010 + + // end block flags + endArc5NotLast = 0x0001 + + // archive encryption block flags + enc5CheckPresent = 0x0001 // password check data is present + + // main archive block flags + arc5MultiVol = 0x0001 + arc5Solid = 0x0004 + + // file block flags + file5IsDir = 0x0001 + file5HasUnixMtime = 0x0002 + file5HasCRC32 = 0x0004 + file5UnpSizeUnknown = 0x0008 + + // file encryption record flags + file5EncCheckPresent = 0x0001 // password check data is present + file5EncUseMac = 0x0002 // use MAC instead of plain checksum + + cacheSize50 = 4 + maxPbkdf2Salt = 64 + pwCheckSize = 8 + maxKdfCount = 24 + + minHeaderSize = 7 +) + +var ( + errBadPassword = errors.New("rardecode: incorrect password") + errCorruptEncrypt = errors.New("rardecode: corrupt encryption data") + errUnknownEncMethod = errors.New("rardecode: unknown encryption method") +) + +type extra struct { + ftype uint64 // field type + data readBuf // field data +} + +type blockHeader50 struct { + htype uint64 // block type + flags uint64 + data readBuf // block header data + extra []extra // extra fields + dataSize int64 // size of block data +} + +// leHash32 wraps a hash.Hash32 to return the result of Sum in little +// endian format. +type leHash32 struct { + hash.Hash32 +} + +func (h leHash32) Sum(b []byte) []byte { + s := h.Sum32() + return append(b, byte(s), byte(s>>8), byte(s>>16), byte(s>>24)) +} + +func newLittleEndianCRC32() hash.Hash32 { + return leHash32{crc32.NewIEEE()} +} + +// hash50 implements fileChecksum for RAR 5 archives +type hash50 struct { + hash.Hash // hash file data is written to + sum []byte // file checksum + key []byte // if present used with hmac in calculating checksum from hash +} + +func (h *hash50) valid() bool { + sum := h.Sum(nil) + if len(h.key) > 0 { + mac := hmac.New(sha256.New, h.key) + mac.Write(sum) + sum = mac.Sum(sum[:0]) + if len(h.sum) == 4 { + // CRC32 + for i, v := range sum[4:] { + sum[i&3] ^= v + } + sum = sum[:4] + } + } + return bytes.Equal(sum, h.sum) +} + +// archive50 implements fileBlockReader for RAR 5 file format archives +type archive50 struct { + byteReader // reader for current block data + v *bufio.Reader // reader for current archive volume + pass []byte + blockKey []byte // key used to encrypt blocks + multi bool // archive is multi-volume + solid bool // is a solid archive + checksum hash50 // file checksum + dec decoder // optional decoder used to unpack file + buf readBuf // temporary buffer + keyCache [cacheSize50]struct { // encryption key cache + kdfCount int + salt []byte + keys [][]byte + } +} + +// calcKeys50 calculates the keys used in RAR 5 archive processing. +// The returned slice of byte slices contains 3 keys. +// Key 0 is used for block or file decryption. +// Key 1 is optionally used for file checksum calculation. +// Key 2 is optionally used for password checking. +func calcKeys50(pass, salt []byte, kdfCount int) [][]byte { + if len(salt) > maxPbkdf2Salt { + salt = salt[:maxPbkdf2Salt] + } + keys := make([][]byte, 3) + if len(keys) == 0 { + return keys + } + + prf := hmac.New(sha256.New, pass) + prf.Write(salt) + prf.Write([]byte{0, 0, 0, 1}) + + t := prf.Sum(nil) + u := append([]byte(nil), t...) + + kdfCount-- + + for i, iter := range []int{kdfCount, 16, 16} { + for iter > 0 { + prf.Reset() + prf.Write(u) + u = prf.Sum(u[:0]) + for j := range u { + t[j] ^= u[j] + } + iter-- + } + keys[i] = append([]byte(nil), t...) + } + + pwcheck := keys[2] + for i, v := range pwcheck[pwCheckSize:] { + pwcheck[i&(pwCheckSize-1)] ^= v + } + keys[2] = pwcheck[:pwCheckSize] + + return keys +} + +// getKeys reads kdfcount and salt from b and returns the corresponding encryption keys. +func (a *archive50) getKeys(b *readBuf) (keys [][]byte, err error) { + if len(*b) < 17 { + return nil, errCorruptEncrypt + } + // read kdf count and salt + kdfCount := int(b.byte()) + if kdfCount > maxKdfCount { + return nil, errCorruptEncrypt + } + kdfCount = 1 << uint(kdfCount) + salt := b.bytes(16) + + // check cache of keys for match + for _, v := range a.keyCache { + if kdfCount == v.kdfCount && bytes.Equal(salt, v.salt) { + return v.keys, nil + } + } + // not found, calculate keys + keys = calcKeys50(a.pass, salt, kdfCount) + + // store in cache + copy(a.keyCache[1:], a.keyCache[:]) + a.keyCache[0].kdfCount = kdfCount + a.keyCache[0].salt = append([]byte(nil), salt...) + a.keyCache[0].keys = keys + + return keys, nil +} + +// checkPassword calculates if a password is correct given password check data and keys. +func checkPassword(b *readBuf, keys [][]byte) error { + if len(*b) < 12 { + return nil // not enough bytes, ignore for the moment + } + pwcheck := b.bytes(8) + sum := b.bytes(4) + csum := sha256.Sum256(pwcheck) + if bytes.Equal(sum, csum[:len(sum)]) && !bytes.Equal(pwcheck, keys[2]) { + return errBadPassword + } + return nil +} + +// parseFileEncryptionRecord processes the optional file encryption record from a file header. +func (a *archive50) parseFileEncryptionRecord(b readBuf, f *fileBlockHeader) error { + if ver := b.uvarint(); ver != 0 { + return errUnknownEncMethod + } + flags := b.uvarint() + + keys, err := a.getKeys(&b) + if err != nil { + return err + } + + f.key = keys[0] + if len(b) < 16 { + return errCorruptEncrypt + } + f.iv = b.bytes(16) + + if flags&file5EncCheckPresent > 0 { + if err := checkPassword(&b, keys); err != nil { + return err + } + } + if flags&file5EncUseMac > 0 { + a.checksum.key = keys[1] + } + return nil +} + +func (a *archive50) parseFileHeader(h *blockHeader50) (*fileBlockHeader, error) { + a.checksum.sum = nil + a.checksum.key = nil + + f := new(fileBlockHeader) + + f.first = h.flags&block5DataNotFirst == 0 + f.last = h.flags&block5DataNotLast == 0 + + flags := h.data.uvarint() // file flags + f.IsDir = flags&file5IsDir > 0 + f.UnKnownSize = flags&file5UnpSizeUnknown > 0 + f.UnPackedSize = int64(h.data.uvarint()) + f.PackedSize = h.dataSize + f.Attributes = int64(h.data.uvarint()) + if flags&file5HasUnixMtime > 0 { + if len(h.data) < 4 { + return nil, errCorruptFileHeader + } + f.ModificationTime = time.Unix(int64(h.data.uint32()), 0) + } + if flags&file5HasCRC32 > 0 { + if len(h.data) < 4 { + return nil, errCorruptFileHeader + } + a.checksum.sum = append([]byte(nil), h.data.bytes(4)...) + if f.first { + a.checksum.Hash = newLittleEndianCRC32() + f.cksum = &a.checksum + } + } + + flags = h.data.uvarint() // compression flags + f.solid = flags&0x0040 > 0 + f.winSize = uint(flags&0x3C00)>>10 + 17 + method := (flags >> 7) & 7 // compression method (0 == none) + if f.first && method != 0 { + unpackver := flags & 0x003f + if unpackver != 0 { + return nil, errUnknownDecoder + } + if a.dec == nil { + a.dec = new(decoder50) + } + f.decoder = a.dec + } + switch h.data.uvarint() { + case 0: + f.HostOS = HostOSWindows + case 1: + f.HostOS = HostOSUnix + default: + f.HostOS = HostOSUnknown + } + nlen := int(h.data.uvarint()) + if len(h.data) < nlen { + return nil, errCorruptFileHeader + } + f.Name = string(h.data.bytes(nlen)) + + // parse optional extra records + for _, e := range h.extra { + var err error + switch e.ftype { + case 1: // encryption + err = a.parseFileEncryptionRecord(e.data, f) + case 2: + // TODO: hash + case 3: + // TODO: time + case 4: // version + _ = e.data.uvarint() // ignore flags field + f.Version = int(e.data.uvarint()) + case 5: + // TODO: redirection + case 6: + // TODO: owner + } + if err != nil { + return nil, err + } + } + return f, nil +} + +// parseEncryptionBlock calculates the key for block encryption. +func (a *archive50) parseEncryptionBlock(b readBuf) error { + if ver := b.uvarint(); ver != 0 { + return errUnknownEncMethod + } + flags := b.uvarint() + keys, err := a.getKeys(&b) + if err != nil { + return err + } + if flags&enc5CheckPresent > 0 { + if err := checkPassword(&b, keys); err != nil { + return err + } + } + a.blockKey = keys[0] + return nil +} + +func (a *archive50) readBlockHeader() (*blockHeader50, error) { + r := io.Reader(a.v) + if a.blockKey != nil { + // block is encrypted + iv := a.buf[:16] + if err := readFull(r, iv); err != nil { + return nil, err + } + r = newAesDecryptReader(r, a.blockKey, iv) + } + + b := a.buf[:minHeaderSize] + if err := readFull(r, b); err != nil { + return nil, err + } + crc := b.uint32() + + hash := crc32.NewIEEE() + hash.Write(b) + + size := int(b.uvarint()) // header size + if size > cap(a.buf) { + a.buf = readBuf(make([]byte, size)) + } else { + a.buf = a.buf[:size] + } + n := copy(a.buf, b) // copy left over bytes + if err := readFull(r, a.buf[n:]); err != nil { // read rest of header + return nil, err + } + + // check header crc + hash.Write(a.buf[n:]) + if crc != hash.Sum32() { + return nil, errBadHeaderCrc + } + + b = a.buf + h := new(blockHeader50) + h.htype = b.uvarint() + h.flags = b.uvarint() + + var extraSize int + if h.flags&block5HasExtra > 0 { + extraSize = int(b.uvarint()) + } + if h.flags&block5HasData > 0 { + h.dataSize = int64(b.uvarint()) + } + if len(b) < extraSize { + return nil, errCorruptHeader + } + h.data = b.bytes(len(b) - extraSize) + + // read header extra records + for len(b) > 0 { + size = int(b.uvarint()) + if len(b) < size { + return nil, errCorruptHeader + } + data := readBuf(b.bytes(size)) + ftype := data.uvarint() + h.extra = append(h.extra, extra{ftype, data}) + } + + return h, nil +} + +// next advances to the next file block in the archive +func (a *archive50) next() (*fileBlockHeader, error) { + for { + h, err := a.readBlockHeader() + if err != nil { + return nil, err + } + a.byteReader = limitByteReader(a.v, h.dataSize) + switch h.htype { + case block5File: + return a.parseFileHeader(h) + case block5Arc: + flags := h.data.uvarint() + a.multi = flags&arc5MultiVol > 0 + a.solid = flags&arc5Solid > 0 + case block5Encrypt: + err = a.parseEncryptionBlock(h.data) + case block5End: + flags := h.data.uvarint() + if flags&endArc5NotLast == 0 || !a.multi { + return nil, errArchiveEnd + } + return nil, errArchiveContinues + default: + // discard block data + _, err = io.Copy(ioutil.Discard, a.byteReader) + } + if err != nil { + return nil, err + } + } +} + +func (a *archive50) version() int { return fileFmt50 } + +func (a *archive50) reset() { + a.blockKey = nil // reset encryption when opening new volume file +} + +func (a *archive50) isSolid() bool { + return a.solid +} + +// newArchive50 creates a new fileBlockReader for a Version 5 archive. +func newArchive50(r *bufio.Reader, password string) fileBlockReader { + a := new(archive50) + a.v = r + a.pass = []byte(password) + a.buf = make([]byte, 100) + return a +} diff --git a/vendor/github.com/nwaples/rardecode/bit_reader.go b/vendor/github.com/nwaples/rardecode/bit_reader.go new file mode 100644 index 0000000..9b284ef --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/bit_reader.go @@ -0,0 +1,119 @@ +package rardecode + +import "io" + +type bitReader interface { + readBits(n uint) (int, error) // read n bits of data + unreadBits(n uint) // revert the reading of the last n bits read +} + +type limitedBitReader struct { + br bitReader + n int + err error // error to return if br returns EOF before all n bits have been read +} + +// limitBitReader returns a bitReader that reads from br and stops with io.EOF after n bits. +// If br returns an io.EOF before reading n bits, err is returned. +func limitBitReader(br bitReader, n int, err error) bitReader { + return &limitedBitReader{br, n, err} +} + +func (l *limitedBitReader) readBits(n uint) (int, error) { + if int(n) > l.n { + return 0, io.EOF + } + v, err := l.br.readBits(n) + if err == nil { + l.n -= int(n) + } else if err == io.EOF { + err = l.err + } + return v, err +} + +func (l *limitedBitReader) unreadBits(n uint) { + l.n += int(n) + l.br.unreadBits(n) +} + +// rarBitReader wraps an io.ByteReader to perform various bit and byte +// reading utility functions used in RAR file processing. +type rarBitReader struct { + r io.ByteReader + v int + n uint +} + +func (r *rarBitReader) reset(br io.ByteReader) { + r.r = br + r.n = 0 + r.v = 0 +} + +func (r *rarBitReader) readBits(n uint) (int, error) { + for n > r.n { + c, err := r.r.ReadByte() + if err != nil { + return 0, err + } + r.v = r.v<<8 | int(c) + r.n += 8 + } + r.n -= n + return (r.v >> r.n) & ((1 << n) - 1), nil +} + +func (r *rarBitReader) unreadBits(n uint) { + r.n += n +} + +// alignByte aligns the current bit reading input to the next byte boundary. +func (r *rarBitReader) alignByte() { + r.n -= r.n % 8 +} + +// readUint32 reads a RAR V3 encoded uint32 +func (r *rarBitReader) readUint32() (uint32, error) { + n, err := r.readBits(2) + if err != nil { + return 0, err + } + if n != 1 { + n, err = r.readBits(4 << uint(n)) + return uint32(n), err + } + n, err = r.readBits(4) + if err != nil { + return 0, err + } + if n == 0 { + n, err = r.readBits(8) + n |= -1 << 8 + return uint32(n), err + } + nlow, err := r.readBits(4) + n = n<<4 | nlow + return uint32(n), err +} + +func (r *rarBitReader) ReadByte() (byte, error) { + n, err := r.readBits(8) + return byte(n), err +} + +// readFull reads len(p) bytes into p. If fewer bytes are read an error is returned. +func (r *rarBitReader) readFull(p []byte) error { + for i := range p { + c, err := r.ReadByte() + if err != nil { + return err + } + p[i] = c + } + return nil +} + +func newRarBitReader(r io.ByteReader) *rarBitReader { + return &rarBitReader{r: r} +} diff --git a/vendor/github.com/nwaples/rardecode/decode29.go b/vendor/github.com/nwaples/rardecode/decode29.go new file mode 100644 index 0000000..638645e --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/decode29.go @@ -0,0 +1,264 @@ +package rardecode + +import ( + "bytes" + "errors" + "io" +) + +const ( + maxCodeSize = 0x10000 + maxUniqueFilters = 1024 +) + +var ( + // Errors marking the end of the decoding block and/or file + endOfFile = errors.New("rardecode: end of file") + endOfBlock = errors.New("rardecode: end of block") + endOfBlockAndFile = errors.New("rardecode: end of block and file") +) + +// decoder29 implements the decoder interface for RAR 3.0 compression (unpack version 29) +// Decode input is broken up into 1 or more blocks. The start of each block specifies +// the decoding algorithm (ppm or lz) and optional data to initialize with. +// Block length is not stored, it is determined only after decoding an end of file and/or +// block marker in the data. +type decoder29 struct { + br *rarBitReader + eof bool // at file eof + fnum int // current filter number (index into filters) + flen []int // filter block length history + filters []v3Filter // list of current filters used by archive encoding + + // current decode function (lz or ppm). + // When called it should perform a single decode operation, and either apply the + // data to the window or return they raw bytes for a filter. + decode func(w *window) ([]byte, error) + + lz lz29Decoder // lz decoder + ppm ppm29Decoder // ppm decoder +} + +// init intializes the decoder for decoding a new file. +func (d *decoder29) init(r io.ByteReader, reset bool) error { + if d.br == nil { + d.br = newRarBitReader(r) + } else { + d.br.reset(r) + } + d.eof = false + if reset { + d.initFilters() + d.lz.reset() + d.ppm.reset() + d.decode = nil + } + if d.decode == nil { + return d.readBlockHeader() + } + return nil +} + +func (d *decoder29) initFilters() { + d.fnum = 0 + d.flen = nil + d.filters = nil +} + +// readVMCode reads the raw bytes for the code/commands used in a vm filter +func readVMCode(br *rarBitReader) ([]byte, error) { + n, err := br.readUint32() + if err != nil { + return nil, err + } + if n > maxCodeSize || n == 0 { + return nil, errInvalidFilter + } + buf := make([]byte, n) + err = br.readFull(buf) + if err != nil { + return nil, err + } + var x byte + for _, c := range buf[1:] { + x ^= c + } + // simple xor checksum on data + if x != buf[0] { + return nil, errInvalidFilter + } + return buf, nil +} + +func (d *decoder29) parseVMFilter(buf []byte) (*filterBlock, error) { + flags := buf[0] + br := newRarBitReader(bytes.NewReader(buf[1:])) + fb := new(filterBlock) + + // Find the filter number which is an index into d.filters. + // If filter number == len(d.filters) it is a new filter to be added. + if flags&0x80 > 0 { + n, err := br.readUint32() + if err != nil { + return nil, err + } + if n == 0 { + d.initFilters() + fb.reset = true + } else { + n-- + if n > maxUniqueFilters { + return nil, errInvalidFilter + } + if int(n) > len(d.filters) { + return nil, errInvalidFilter + } + } + d.fnum = int(n) + } + + // filter offset + n, err := br.readUint32() + if err != nil { + return nil, err + } + if flags&0x40 > 0 { + n += 258 + } + fb.offset = int(n) + + // filter length + if d.fnum == len(d.flen) { + d.flen = append(d.flen, 0) + } + if flags&0x20 > 0 { + n, err = br.readUint32() + if err != nil { + return nil, err + } + //fb.length = int(n) + d.flen[d.fnum] = int(n) + } + fb.length = d.flen[d.fnum] + + // initial register values + r := make(map[int]uint32) + if flags&0x10 > 0 { + bits, err := br.readBits(vmRegs - 1) + if err != nil { + return nil, err + } + for i := 0; i < vmRegs-1; i++ { + if bits&1 > 0 { + r[i], err = br.readUint32() + if err != nil { + return nil, err + } + } + bits >>= 1 + } + } + + // filter is new so read the code for it + if d.fnum == len(d.filters) { + code, err := readVMCode(br) + if err != nil { + return nil, err + } + f, err := getV3Filter(code) + if err != nil { + return nil, err + } + d.filters = append(d.filters, f) + d.flen = append(d.flen, fb.length) + } + + // read global data + var g []byte + if flags&0x08 > 0 { + n, err := br.readUint32() + if err != nil { + return nil, err + } + if n > vmGlobalSize-vmFixedGlobalSize { + return nil, errInvalidFilter + } + g = make([]byte, n) + err = br.readFull(g) + if err != nil { + return nil, err + } + } + + // create filter function + f := d.filters[d.fnum] + fb.filter = func(buf []byte, offset int64) ([]byte, error) { + return f(r, g, buf, offset) + } + + return fb, nil +} + +// readBlockHeader determines and initializes the current decoder for a new decode block. +func (d *decoder29) readBlockHeader() error { + d.br.alignByte() + n, err := d.br.readBits(1) + if err == nil { + if n > 0 { + d.decode = d.ppm.decode + err = d.ppm.init(d.br) + } else { + d.decode = d.lz.decode + err = d.lz.init(d.br) + } + } + if err == io.EOF { + err = errDecoderOutOfData + } + return err + +} + +func (d *decoder29) fill(w *window) ([]*filterBlock, error) { + if d.eof { + return nil, io.EOF + } + + var fl []*filterBlock + + for w.available() > 0 { + b, err := d.decode(w) // perform a single decode operation + if len(b) > 0 && err == nil { + // parse raw data for filter and add to list of filters + var f *filterBlock + f, err = d.parseVMFilter(b) + if f != nil { + // make offset relative to read index (from write index) + f.offset += w.buffered() + fl = append(fl, f) + } + } + + switch err { + case nil: + continue + case endOfBlock: + err = d.readBlockHeader() + if err == nil { + continue + } + case endOfFile: + d.eof = true + err = io.EOF + case endOfBlockAndFile: + d.eof = true + d.decode = nil // clear decoder, it will be setup by next init() + err = io.EOF + case io.EOF: + err = errDecoderOutOfData + } + return fl, err + } + // return filters + return fl, nil +} diff --git a/vendor/github.com/nwaples/rardecode/decode29_lz.go b/vendor/github.com/nwaples/rardecode/decode29_lz.go new file mode 100644 index 0000000..9447085 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/decode29_lz.go @@ -0,0 +1,247 @@ +package rardecode + +const ( + mainSize = 299 + offsetSize = 60 + lowOffsetSize = 17 + lengthSize = 28 + tableSize = mainSize + offsetSize + lowOffsetSize + lengthSize +) + +var ( + lengthBase = [28]int{0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, + 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224} + lengthExtraBits = [28]uint{0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, + 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5} + + offsetBase = [60]int{0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, + 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096, + 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, + 131072, 196608, 262144, 327680, 393216, 458752, 524288, + 589824, 655360, 720896, 786432, 851968, 917504, 983040, + 1048576, 1310720, 1572864, 1835008, 2097152, 2359296, 2621440, + 2883584, 3145728, 3407872, 3670016, 3932160} + offsetExtraBits = [60]uint{0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, + 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18} + + shortOffsetBase = [8]int{0, 4, 8, 16, 32, 64, 128, 192} + shortOffsetExtraBits = [8]uint{2, 2, 3, 4, 5, 6, 6, 6} +) + +type lz29Decoder struct { + codeLength [tableSize]byte + + mainDecoder huffmanDecoder + offsetDecoder huffmanDecoder + lowOffsetDecoder huffmanDecoder + lengthDecoder huffmanDecoder + + offset [4]int // history of previous offsets + length int // previous length + lowOffset int + lowOffsetRepeats int + + br *rarBitReader +} + +func (d *lz29Decoder) reset() { + for i := range d.offset { + d.offset[i] = 0 + } + d.length = 0 + for i := range d.codeLength { + d.codeLength[i] = 0 + } +} + +func (d *lz29Decoder) init(br *rarBitReader) error { + d.br = br + d.lowOffset = 0 + d.lowOffsetRepeats = 0 + + n, err := d.br.readBits(1) + if err != nil { + return err + } + addOld := n > 0 + + cl := d.codeLength[:] + if err = readCodeLengthTable(d.br, cl, addOld); err != nil { + return err + } + + d.mainDecoder.init(cl[:mainSize]) + cl = cl[mainSize:] + d.offsetDecoder.init(cl[:offsetSize]) + cl = cl[offsetSize:] + d.lowOffsetDecoder.init(cl[:lowOffsetSize]) + cl = cl[lowOffsetSize:] + d.lengthDecoder.init(cl) + + return nil +} + +func (d *lz29Decoder) readFilterData() (b []byte, err error) { + flags, err := d.br.ReadByte() + if err != nil { + return nil, err + } + + n := (int(flags) & 7) + 1 + switch n { + case 7: + n, err = d.br.readBits(8) + n += 7 + if err != nil { + return nil, err + } + case 8: + n, err = d.br.readBits(16) + if err != nil { + return nil, err + } + } + + buf := make([]byte, n+1) + buf[0] = flags + err = d.br.readFull(buf[1:]) + + return buf, err +} + +func (d *lz29Decoder) readEndOfBlock() error { + n, err := d.br.readBits(1) + if err != nil { + return err + } + if n > 0 { + return endOfBlock + } + n, err = d.br.readBits(1) + if err != nil { + return err + } + if n > 0 { + return endOfBlockAndFile + } + return endOfFile +} + +func (d *lz29Decoder) decode(win *window) ([]byte, error) { + sym, err := d.mainDecoder.readSym(d.br) + if err != nil { + return nil, err + } + + switch { + case sym < 256: + // literal + win.writeByte(byte(sym)) + return nil, nil + case sym == 256: + return nil, d.readEndOfBlock() + case sym == 257: + return d.readFilterData() + case sym == 258: + // use previous offset and length + case sym < 263: + i := sym - 259 + offset := d.offset[i] + copy(d.offset[1:i+1], d.offset[:i]) + d.offset[0] = offset + + i, err := d.lengthDecoder.readSym(d.br) + if err != nil { + return nil, err + } + d.length = lengthBase[i] + 2 + bits := lengthExtraBits[i] + if bits > 0 { + n, err := d.br.readBits(bits) + if err != nil { + return nil, err + } + d.length += n + } + case sym < 271: + i := sym - 263 + copy(d.offset[1:], d.offset[:]) + offset := shortOffsetBase[i] + 1 + bits := shortOffsetExtraBits[i] + if bits > 0 { + n, err := d.br.readBits(bits) + if err != nil { + return nil, err + } + offset += n + } + d.offset[0] = offset + + d.length = 2 + default: + i := sym - 271 + d.length = lengthBase[i] + 3 + bits := lengthExtraBits[i] + if bits > 0 { + n, err := d.br.readBits(bits) + if err != nil { + return nil, err + } + d.length += n + } + + i, err = d.offsetDecoder.readSym(d.br) + if err != nil { + return nil, err + } + offset := offsetBase[i] + 1 + bits = offsetExtraBits[i] + + switch { + case bits >= 4: + if bits > 4 { + n, err := d.br.readBits(bits - 4) + if err != nil { + return nil, err + } + offset += n << 4 + } + + if d.lowOffsetRepeats > 0 { + d.lowOffsetRepeats-- + offset += d.lowOffset + } else { + n, err := d.lowOffsetDecoder.readSym(d.br) + if err != nil { + return nil, err + } + if n == 16 { + d.lowOffsetRepeats = 15 + offset += d.lowOffset + } else { + offset += n + d.lowOffset = n + } + } + case bits > 0: + n, err := d.br.readBits(bits) + if err != nil { + return nil, err + } + offset += n + } + + if offset >= 0x2000 { + d.length++ + if offset >= 0x40000 { + d.length++ + } + } + copy(d.offset[1:], d.offset[:]) + d.offset[0] = offset + } + win.copyBytes(d.length, d.offset[0]) + return nil, nil +} diff --git a/vendor/github.com/nwaples/rardecode/decode29_ppm.go b/vendor/github.com/nwaples/rardecode/decode29_ppm.go new file mode 100644 index 0000000..39c3199 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/decode29_ppm.go @@ -0,0 +1,132 @@ +package rardecode + +import "io" + +type ppm29Decoder struct { + m model // ppm model + esc byte // escape character + br io.ByteReader +} + +func (d *ppm29Decoder) init(br *rarBitReader) error { + maxOrder, err := br.readBits(7) + if err != nil { + return err + } + reset := maxOrder&0x20 > 0 + + // Should have flushed all unread bits from bitReader by now, + // use underlying ByteReader + d.br = br.r + + var maxMB int + if reset { + c, err := d.br.ReadByte() + if err != nil { + return err + } + maxMB = int(c) + 1 + } + + if maxOrder&0x40 > 0 { + d.esc, err = d.br.ReadByte() + if err != nil { + return err + } + } + + maxOrder = (maxOrder & 0x1f) + 1 + if maxOrder > 16 { + maxOrder = 16 + (maxOrder-16)*3 + } + + return d.m.init(d.br, reset, maxOrder, maxMB) +} + +func (d *ppm29Decoder) reset() { + d.esc = 2 +} + +func (d *ppm29Decoder) readFilterData() ([]byte, error) { + c, err := d.m.ReadByte() + if err != nil { + return nil, err + } + n := int(c&7) + 1 + if n == 7 { + b, err := d.m.ReadByte() + if err != nil { + return nil, err + } + n += int(b) + } else if n == 8 { + b, err := d.m.ReadByte() + if err != nil { + return nil, err + } + n = int(b) << 8 + b, err = d.m.ReadByte() + if err != nil { + return nil, err + } + n |= int(b) + } + + n++ + buf := make([]byte, n) + buf[0] = byte(c) + for i := 1; i < n; i++ { + buf[i], err = d.m.ReadByte() + if err != nil { + return nil, err + } + } + return buf, nil +} + +func (d *ppm29Decoder) decode(w *window) ([]byte, error) { + c, err := d.m.ReadByte() + if err != nil { + return nil, err + } + if c != d.esc { + w.writeByte(c) + return nil, nil + } + c, err = d.m.ReadByte() + if err != nil { + return nil, err + } + + switch c { + case 0: + return nil, endOfBlock + case 2: + return nil, endOfBlockAndFile + case 3: + return d.readFilterData() + case 4: + offset := 0 + for i := 0; i < 3; i++ { + c, err = d.m.ReadByte() + if err != nil { + return nil, err + } + offset = offset<<8 | int(c) + } + len, err := d.m.ReadByte() + if err != nil { + return nil, err + } + w.copyBytes(int(len)+32, offset+2) + case 5: + len, err := d.m.ReadByte() + if err != nil { + return nil, err + } + w.copyBytes(int(len)+4, 1) + default: + w.writeByte(d.esc) + } + return nil, nil +} diff --git a/vendor/github.com/nwaples/rardecode/decode50.go b/vendor/github.com/nwaples/rardecode/decode50.go new file mode 100644 index 0000000..1939a44 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/decode50.go @@ -0,0 +1,294 @@ +package rardecode + +import ( + "errors" + "io" +) + +const ( + mainSize5 = 306 + offsetSize5 = 64 + lowoffsetSize5 = 16 + lengthSize5 = 44 + tableSize5 = mainSize5 + offsetSize5 + lowoffsetSize5 + lengthSize5 +) + +var ( + errUnknownFilter = errors.New("rardecode: unknown V5 filter") + errCorruptDecodeHeader = errors.New("rardecode: corrupt decode header") +) + +// decoder50 implements the decoder interface for RAR 5 compression. +// Decode input it broken up into 1 or more blocks. Each block starts with +// a header containing block length and optional code length tables to initialize +// the huffman decoders with. +type decoder50 struct { + r io.ByteReader + br bitReader // bit reader for current data block + codeLength [tableSize5]byte + + lastBlock bool // current block is last block in compressed file + + mainDecoder huffmanDecoder + offsetDecoder huffmanDecoder + lowoffsetDecoder huffmanDecoder + lengthDecoder huffmanDecoder + + offset [4]int + length int +} + +func (d *decoder50) init(r io.ByteReader, reset bool) error { + d.r = r + d.lastBlock = false + + if reset { + for i := range d.offset { + d.offset[i] = 0 + } + d.length = 0 + for i := range d.codeLength { + d.codeLength[i] = 0 + } + } + err := d.readBlockHeader() + if err == io.EOF { + return errDecoderOutOfData + } + return err +} + +func (d *decoder50) readBlockHeader() error { + flags, err := d.r.ReadByte() + if err != nil { + return err + } + + bytecount := (flags>>3)&3 + 1 + if bytecount == 4 { + return errCorruptDecodeHeader + } + + hsum, err := d.r.ReadByte() + if err != nil { + return err + } + + blockBits := int(flags)&0x07 + 1 + blockBytes := 0 + sum := 0x5a ^ flags + for i := byte(0); i < bytecount; i++ { + n, err := d.r.ReadByte() + if err != nil { + return err + } + sum ^= n + blockBytes |= int(n) << (i * 8) + } + if sum != hsum { // bad header checksum + return errCorruptDecodeHeader + } + blockBits += (blockBytes - 1) * 8 + + // create bit reader for block + d.br = limitBitReader(newRarBitReader(d.r), blockBits, errDecoderOutOfData) + d.lastBlock = flags&0x40 > 0 + + if flags&0x80 > 0 { + // read new code length tables and reinitialize huffman decoders + cl := d.codeLength[:] + err = readCodeLengthTable(d.br, cl, false) + if err != nil { + return err + } + d.mainDecoder.init(cl[:mainSize5]) + cl = cl[mainSize5:] + d.offsetDecoder.init(cl[:offsetSize5]) + cl = cl[offsetSize5:] + d.lowoffsetDecoder.init(cl[:lowoffsetSize5]) + cl = cl[lowoffsetSize5:] + d.lengthDecoder.init(cl) + } + return nil +} + +func slotToLength(br bitReader, n int) (int, error) { + if n >= 8 { + bits := uint(n/4 - 1) + n = (4 | (n & 3)) << bits + if bits > 0 { + b, err := br.readBits(bits) + if err != nil { + return 0, err + } + n |= b + } + } + n += 2 + return n, nil +} + +// readFilter5Data reads an encoded integer used in V5 filters. +func readFilter5Data(br bitReader) (int, error) { + // TODO: should data really be uint? (for 32bit ints). + // It will be masked later anyway by decode window mask. + bytes, err := br.readBits(2) + if err != nil { + return 0, err + } + bytes++ + + var data int + for i := 0; i < bytes; i++ { + n, err := br.readBits(8) + if err != nil { + return 0, err + } + data |= n << (uint(i) * 8) + } + return data, nil +} + +func readFilter(br bitReader) (*filterBlock, error) { + fb := new(filterBlock) + var err error + + fb.offset, err = readFilter5Data(br) + if err != nil { + return nil, err + } + fb.length, err = readFilter5Data(br) + if err != nil { + return nil, err + } + ftype, err := br.readBits(3) + if err != nil { + return nil, err + } + switch ftype { + case 0: + n, err := br.readBits(5) + if err != nil { + return nil, err + } + fb.filter = func(buf []byte, offset int64) ([]byte, error) { return filterDelta(n+1, buf) } + case 1: + fb.filter = func(buf []byte, offset int64) ([]byte, error) { return filterE8(0xe8, true, buf, offset) } + case 2: + fb.filter = func(buf []byte, offset int64) ([]byte, error) { return filterE8(0xe9, true, buf, offset) } + case 3: + fb.filter = filterArm + default: + return nil, errUnknownFilter + } + return fb, nil +} + +func (d *decoder50) decodeSym(win *window, sym int) (*filterBlock, error) { + switch { + case sym < 256: + // literal + win.writeByte(byte(sym)) + return nil, nil + case sym == 256: + f, err := readFilter(d.br) + f.offset += win.buffered() + return f, err + case sym == 257: + // use previous offset and length + case sym < 262: + i := sym - 258 + offset := d.offset[i] + copy(d.offset[1:i+1], d.offset[:i]) + d.offset[0] = offset + + sl, err := d.lengthDecoder.readSym(d.br) + if err != nil { + return nil, err + } + d.length, err = slotToLength(d.br, sl) + if err != nil { + return nil, err + } + default: + length, err := slotToLength(d.br, sym-262) + if err != nil { + return nil, err + } + + offset := 1 + slot, err := d.offsetDecoder.readSym(d.br) + if err != nil { + return nil, err + } + if slot < 4 { + offset += slot + } else { + bits := uint(slot/2 - 1) + offset += (2 | (slot & 1)) << bits + + if bits >= 4 { + if bits > 4 { + n, err := d.br.readBits(bits - 4) + if err != nil { + return nil, err + } + offset += n << 4 + } + n, err := d.lowoffsetDecoder.readSym(d.br) + if err != nil { + return nil, err + } + offset += n + } else { + n, err := d.br.readBits(bits) + if err != nil { + return nil, err + } + offset += n + } + } + if offset > 0x100 { + length++ + if offset > 0x2000 { + length++ + if offset > 0x40000 { + length++ + } + } + } + copy(d.offset[1:], d.offset[:]) + d.offset[0] = offset + d.length = length + } + win.copyBytes(d.length, d.offset[0]) + return nil, nil +} + +func (d *decoder50) fill(w *window) ([]*filterBlock, error) { + var fl []*filterBlock + + for w.available() > 0 { + sym, err := d.mainDecoder.readSym(d.br) + if err == nil { + var f *filterBlock + f, err = d.decodeSym(w, sym) + if f != nil { + fl = append(fl, f) + } + } else if err == io.EOF { + // reached end of the block + if d.lastBlock { + return fl, io.EOF + } + err = d.readBlockHeader() + } + if err != nil { + if err == io.EOF { + return fl, errDecoderOutOfData + } + return fl, err + } + } + return fl, nil +} diff --git a/vendor/github.com/nwaples/rardecode/decode_reader.go b/vendor/github.com/nwaples/rardecode/decode_reader.go new file mode 100644 index 0000000..b346936 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/decode_reader.go @@ -0,0 +1,290 @@ +package rardecode + +import ( + "errors" + "io" +) + +const ( + minWindowSize = 0x40000 + maxQueuedFilters = 8192 +) + +var ( + errTooManyFilters = errors.New("rardecode: too many filters") + errInvalidFilter = errors.New("rardecode: invalid filter") +) + +// filter functions take a byte slice, the current output offset and +// returns transformed data. +type filter func(b []byte, offset int64) ([]byte, error) + +// filterBlock is a block of data to be processed by a filter. +type filterBlock struct { + length int // length of block + offset int // bytes to be read before start of block + reset bool // drop all existing queued filters + filter filter // filter function +} + +// decoder is the interface for decoding compressed data +type decoder interface { + init(r io.ByteReader, reset bool) error // initialize decoder for current file + fill(w *window) ([]*filterBlock, error) // fill window with decoded data, returning any filters +} + +// window is a sliding window buffer. +type window struct { + buf []byte + mask int // buf length mask + r int // index in buf for reads (beginning) + w int // index in buf for writes (end) + l int // length of bytes to be processed by copyBytes + o int // offset of bytes to be processed by copyBytes +} + +// buffered returns the number of bytes yet to be read from window +func (w *window) buffered() int { return (w.w - w.r) & w.mask } + +// available returns the number of bytes that can be written before the window is full +func (w *window) available() int { return (w.r - w.w - 1) & w.mask } + +func (w *window) reset(log2size uint, clear bool) { + size := 1 << log2size + if size < minWindowSize { + size = minWindowSize + } + if size > len(w.buf) { + b := make([]byte, size) + if clear { + w.w = 0 + } else if len(w.buf) > 0 { + n := copy(b, w.buf[w.w:]) + n += copy(b[n:], w.buf[:w.w]) + w.w = n + } + w.buf = b + w.mask = size - 1 + } else if clear { + for i := range w.buf { + w.buf[i] = 0 + } + w.w = 0 + } + w.r = w.w +} + +// writeByte writes c to the end of the window +func (w *window) writeByte(c byte) { + w.buf[w.w] = c + w.w = (w.w + 1) & w.mask +} + +// copyBytes copies len bytes at off distance from the end +// to the end of the window. +func (w *window) copyBytes(len, off int) { + len &= w.mask + + n := w.available() + if len > n { + // if there is not enough space availaible we copy + // as much as we can and save the offset and length + // of the remaining data to be copied later. + w.l = len - n + w.o = off + len = n + } + + i := (w.w - off) & w.mask + for ; len > 0; len-- { + w.buf[w.w] = w.buf[i] + w.w = (w.w + 1) & w.mask + i = (i + 1) & w.mask + } +} + +// read reads bytes from the beginning of the window into p +func (w *window) read(p []byte) (n int) { + if w.r > w.w { + n = copy(p, w.buf[w.r:]) + w.r = (w.r + n) & w.mask + p = p[n:] + } + if w.r < w.w { + l := copy(p, w.buf[w.r:w.w]) + w.r += l + n += l + } + if w.l > 0 && n > 0 { + // if we have successfully read data, copy any + // leftover data from a previous copyBytes. + l := w.l + w.l = 0 + w.copyBytes(l, w.o) + } + return n +} + +// decodeReader implements io.Reader for decoding compressed data in RAR archives. +type decodeReader struct { + win window // sliding window buffer used as decode dictionary + dec decoder // decoder being used to unpack file + tot int64 // total bytes read + buf []byte // filter input/output buffer + outbuf []byte // filter output not yet read + err error + filters []*filterBlock // list of filterBlock's, each with offset relative to previous in list +} + +func (d *decodeReader) init(r io.ByteReader, dec decoder, winsize uint, reset bool) error { + if reset { + d.filters = nil + } + d.err = nil + d.outbuf = nil + d.tot = 0 + d.win.reset(winsize, reset) + d.dec = dec + return d.dec.init(r, reset) +} + +func (d *decodeReader) readErr() error { + err := d.err + d.err = nil + return err +} + +// queueFilter adds a filterBlock to the end decodeReader's filters. +func (d *decodeReader) queueFilter(f *filterBlock) error { + if f.reset { + d.filters = nil + } + if len(d.filters) >= maxQueuedFilters { + return errTooManyFilters + } + // offset & length must be < window size + f.offset &= d.win.mask + f.length &= d.win.mask + // make offset relative to previous filter in list + for _, fb := range d.filters { + if f.offset < fb.offset { + // filter block must not start before previous filter + return errInvalidFilter + } + f.offset -= fb.offset + } + d.filters = append(d.filters, f) + return nil +} + +// processFilters processes any filters valid at the current read index +// and stores the output in outbuf. +func (d *decodeReader) processFilters() (err error) { + f := d.filters[0] + if f.offset > 0 { + return nil + } + d.filters = d.filters[1:] + if d.win.buffered() < f.length { + // fill() didn't return enough bytes + err = d.readErr() + if err == nil || err == io.EOF { + return errInvalidFilter + } + return err + } + + if cap(d.buf) < f.length { + d.buf = make([]byte, f.length) + } + d.outbuf = d.buf[:f.length] + n := d.win.read(d.outbuf) + for { + // run filter passing buffer and total bytes read so far + d.outbuf, err = f.filter(d.outbuf, d.tot) + if err != nil { + return err + } + if cap(d.outbuf) > cap(d.buf) { + // Filter returned a bigger buffer, save it for future filters. + d.buf = d.outbuf + } + if len(d.filters) == 0 { + return nil + } + f = d.filters[0] + + if f.offset != 0 { + // next filter not at current offset + f.offset -= n + return nil + } + if f.length != len(d.outbuf) { + return errInvalidFilter + } + d.filters = d.filters[1:] + + if cap(d.outbuf) < cap(d.buf) { + // Filter returned a smaller buffer. Copy it back to the saved buffer + // so the next filter can make use of the larger buffer if needed. + d.outbuf = append(d.buf[:0], d.outbuf...) + } + } +} + +// fill fills the decodeReader's window +func (d *decodeReader) fill() { + if d.err != nil { + return + } + var fl []*filterBlock + fl, d.err = d.dec.fill(&d.win) // fill window using decoder + for _, f := range fl { + err := d.queueFilter(f) + if err != nil { + d.err = err + return + } + } +} + +// Read decodes data and stores it in p. +func (d *decodeReader) Read(p []byte) (n int, err error) { + if len(d.outbuf) == 0 { + // no filter output, see if we need to create more + if d.win.buffered() == 0 { + // fill empty window + d.fill() + if d.win.buffered() == 0 { + return 0, d.readErr() + } + } else if len(d.filters) > 0 { + f := d.filters[0] + if f.offset == 0 && f.length > d.win.buffered() { + d.fill() // filter at current offset needs more data + } + } + if len(d.filters) > 0 { + if err := d.processFilters(); err != nil { + return 0, err + } + } + } + if len(d.outbuf) > 0 { + // copy filter output into p + n = copy(p, d.outbuf) + d.outbuf = d.outbuf[n:] + } else if len(d.filters) > 0 { + f := d.filters[0] + if f.offset < len(p) { + // only read data up to beginning of next filter + p = p[:f.offset] + } + n = d.win.read(p) // read directly from window + f.offset -= n // adjust first filter offset by bytes just read + } else { + n = d.win.read(p) // read directly from window + } + d.tot += int64(n) + return n, nil +} diff --git a/vendor/github.com/nwaples/rardecode/decrypt_reader.go b/vendor/github.com/nwaples/rardecode/decrypt_reader.go new file mode 100644 index 0000000..bb9f279 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/decrypt_reader.go @@ -0,0 +1,126 @@ +package rardecode + +import ( + "crypto/aes" + "crypto/cipher" + "io" +) + +// cipherBlockReader implements Block Mode decryption of an io.Reader object. +type cipherBlockReader struct { + r io.Reader + mode cipher.BlockMode + inbuf []byte // input buffer for partial data block + outbuf []byte // output buffer used when output slice < block size + n int // bytes read from outbuf + err error +} + +// read reads and decrypts one or more input blocks into p. +// len(p) must be >= cipher block size. +func (cr *cipherBlockReader) read(p []byte) (n int, err error) { + bs := cr.mode.BlockSize() + // round p down to a multiple of the block size + l := len(p) - len(p)%bs + p = p[:l] + + l = len(cr.inbuf) + if l > 0 { + // copy any buffered input into p + copy(p, cr.inbuf) + cr.inbuf = cr.inbuf[:0] + } + // read data for at least one block + n, err = io.ReadAtLeast(cr.r, p[l:], bs-l) + n += l + p = p[:n] + + l = n % bs + // check if p is a multiple of the cipher block size + if l > 0 { + n -= l + // save trailing partial block to process later + cr.inbuf = append(cr.inbuf, p[n:]...) + p = p[:n] + } + + if err != nil { + if err == io.ErrUnexpectedEOF || err == io.ErrShortBuffer { + // ignore trailing bytes < block size length + err = io.EOF + } + return 0, err + } + cr.mode.CryptBlocks(p, p) // decrypt block(s) + return n, nil +} + +// Read reads and decrypts data into p. +// If the input is not a multiple of the cipher block size, +// the trailing bytes will be ignored. +func (cr *cipherBlockReader) Read(p []byte) (n int, err error) { + for { + if cr.n < len(cr.outbuf) { + // return buffered output + n = copy(p, cr.outbuf[cr.n:]) + cr.n += n + return n, nil + } + if cr.err != nil { + err = cr.err + cr.err = nil + return 0, err + } + if len(p) >= cap(cr.outbuf) { + break + } + // p is not large enough to process a block, use outbuf instead + n, cr.err = cr.read(cr.outbuf[:cap(cr.outbuf)]) + cr.outbuf = cr.outbuf[:n] + cr.n = 0 + } + // read blocks into p + return cr.read(p) +} + +// ReadByte returns the next decrypted byte. +func (cr *cipherBlockReader) ReadByte() (byte, error) { + for { + if cr.n < len(cr.outbuf) { + c := cr.outbuf[cr.n] + cr.n++ + return c, nil + } + if cr.err != nil { + err := cr.err + cr.err = nil + return 0, err + } + // refill outbuf + var n int + n, cr.err = cr.read(cr.outbuf[:cap(cr.outbuf)]) + cr.outbuf = cr.outbuf[:n] + cr.n = 0 + } +} + +// newCipherBlockReader returns a cipherBlockReader that decrypts the given io.Reader using +// the provided block mode cipher. +func newCipherBlockReader(r io.Reader, mode cipher.BlockMode) *cipherBlockReader { + cr := &cipherBlockReader{r: r, mode: mode} + cr.outbuf = make([]byte, 0, mode.BlockSize()) + cr.inbuf = make([]byte, 0, mode.BlockSize()) + return cr +} + +// newAesDecryptReader returns a cipherBlockReader that decrypts input from a given io.Reader using AES. +// It will panic if the provided key is invalid. +func newAesDecryptReader(r io.Reader, key, iv []byte) *cipherBlockReader { + block, err := aes.NewCipher(key) + if err != nil { + panic(err) + } + mode := cipher.NewCBCDecrypter(block, iv) + + return newCipherBlockReader(r, mode) +} diff --git a/vendor/github.com/nwaples/rardecode/filters.go b/vendor/github.com/nwaples/rardecode/filters.go new file mode 100644 index 0000000..a9eb040 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/filters.go @@ -0,0 +1,416 @@ +package rardecode + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "io" +) + +const ( + fileSize = 0x1000000 + + vmGlobalAddr = 0x3C000 + vmGlobalSize = 0x02000 + vmFixedGlobalSize = 0x40 + + maxUint32 = 1<<32 - 1 +) + +// v3Filter is the interface type for RAR V3 filters. +// v3Filter performs the same function as the filter type, except that it also takes +// the initial register values r, and global data as input for the RAR V3 VM. +type v3Filter func(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) + +var ( + // standardV3Filters is a list of known filters. We can replace the use of a vm + // filter with a custom filter function. + standardV3Filters = []struct { + crc uint32 // crc of code byte slice for filter + len int // length of code byte slice for filter + f v3Filter // replacement filter function + }{ + {0xad576887, 53, e8FilterV3}, + {0x3cd7e57e, 57, e8e9FilterV3}, + {0x3769893f, 120, itaniumFilterV3}, + {0x0e06077d, 29, deltaFilterV3}, + {0x1c2c5dc8, 149, filterRGBV3}, + {0xbc85e701, 216, filterAudioV3}, + } + + // itanium filter byte masks + byteMask = []int{4, 4, 6, 6, 0, 0, 7, 7, 4, 4, 0, 0, 4, 4, 0, 0} +) + +func filterE8(c byte, v5 bool, buf []byte, offset int64) ([]byte, error) { + off := int32(offset) + for b := buf; len(b) >= 5; { + ch := b[0] + b = b[1:] + off++ + if ch != 0xe8 && ch != c { + continue + } + if v5 { + off %= fileSize + } + addr := int32(binary.LittleEndian.Uint32(b)) + if addr < 0 { + if addr+off >= 0 { + binary.LittleEndian.PutUint32(b, uint32(addr+fileSize)) + } + } else if addr < fileSize { + binary.LittleEndian.PutUint32(b, uint32(addr-off)) + } + off += 4 + b = b[4:] + } + return buf, nil +} + +func e8FilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + return filterE8(0xe8, false, buf, offset) +} + +func e8e9FilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + return filterE8(0xe9, false, buf, offset) +} + +func getBits(buf []byte, pos, count uint) uint32 { + n := binary.LittleEndian.Uint32(buf[pos/8:]) + n >>= pos & 7 + mask := uint32(maxUint32) >> (32 - count) + return n & mask +} + +func setBits(buf []byte, pos, count uint, bits uint32) { + mask := uint32(maxUint32) >> (32 - count) + mask <<= pos & 7 + bits <<= pos & 7 + n := binary.LittleEndian.Uint32(buf[pos/8:]) + n = (n & ^mask) | (bits & mask) + binary.LittleEndian.PutUint32(buf[pos/8:], n) +} + +func itaniumFilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + fileOffset := uint32(offset) >> 4 + + for b := buf; len(b) > 21; b = b[16:] { + c := int(b[0]&0x1f) - 0x10 + if c >= 0 { + mask := byteMask[c] + if mask != 0 { + for i := uint(0); i <= 2; i++ { + if mask&(1<= 2*l { + res = buf[l : 2*l] // use unused capacity + } else { + res = make([]byte, l, 2*l) + } + + i := 0 + for j := 0; j < n; j++ { + var c byte + for k := j; k < len(res); k += n { + c -= buf[i] + i++ + res[k] = c + } + } + return res, nil +} + +func deltaFilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + return filterDelta(int(r[0]), buf) +} + +func abs(n int) int { + if n < 0 { + n = -n + } + return n +} + +func filterRGBV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + width := int(r[0] - 3) + posR := int(r[1]) + if posR < 0 || width < 0 { + return buf, nil + } + + var res []byte + l := len(buf) + if cap(buf) >= 2*l { + res = buf[l : 2*l] // use unused capacity + } else { + res = make([]byte, l, 2*l) + } + + for c := 0; c < 3; c++ { + var prevByte int + for i := c; i < len(res); i += 3 { + var predicted int + upperPos := i - width + if upperPos >= 3 { + upperByte := int(res[upperPos]) + upperLeftByte := int(res[upperPos-3]) + predicted = prevByte + upperByte - upperLeftByte + pa := abs(predicted - prevByte) + pb := abs(predicted - upperByte) + pc := abs(predicted - upperLeftByte) + if pa <= pb && pa <= pc { + predicted = prevByte + } else if pb <= pc { + predicted = upperByte + } else { + predicted = upperLeftByte + } + } else { + predicted = prevByte + } + prevByte = (predicted - int(buf[0])) & 0xFF + res[i] = uint8(prevByte) + buf = buf[1:] + } + + } + for i := posR; i < len(res)-2; i += 3 { + c := res[i+1] + res[i] += c + res[i+2] += c + } + return res, nil +} + +func filterAudioV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + var res []byte + l := len(buf) + if cap(buf) >= 2*l { + res = buf[l : 2*l] // use unused capacity + } else { + res = make([]byte, l, 2*l) + } + + chans := int(r[0]) + for c := 0; c < chans; c++ { + var prevByte, byteCount int + var diff [7]int + var d, k [3]int + + for i := c; i < len(res); i += chans { + predicted := prevByte<<3 + k[0]*d[0] + k[1]*d[1] + k[2]*d[2] + predicted = int(int8(predicted >> 3)) + + curByte := int(int8(buf[0])) + buf = buf[1:] + predicted -= curByte + res[i] = uint8(predicted) + + dd := curByte << 3 + diff[0] += abs(dd) + diff[1] += abs(dd - d[0]) + diff[2] += abs(dd + d[0]) + diff[3] += abs(dd - d[1]) + diff[4] += abs(dd + d[1]) + diff[5] += abs(dd - d[2]) + diff[6] += abs(dd + d[2]) + + prevDelta := int(int8(predicted - prevByte)) + prevByte = predicted + d[2] = d[1] + d[1] = prevDelta - d[0] + d[0] = prevDelta + + if byteCount&0x1f == 0 { + min := diff[0] + diff[0] = 0 + n := 0 + for j := 1; j < len(diff); j++ { + if diff[j] < min { + min = diff[j] + n = j + } + diff[j] = 0 + } + n-- + if n >= 0 { + m := n / 2 + if n%2 == 0 { + if k[m] >= -16 { + k[m]-- + } + } else { + if k[m] < 16 { + k[m]++ + } + } + } + } + byteCount++ + } + + } + return res, nil +} + +func filterArm(buf []byte, offset int64) ([]byte, error) { + for i := 0; len(buf)-i > 3; i += 4 { + if buf[i+3] == 0xeb { + n := uint(buf[i]) + n += uint(buf[i+1]) * 0x100 + n += uint(buf[i+2]) * 0x10000 + n -= (uint(offset) + uint(i)) / 4 + buf[i] = byte(n) + buf[i+1] = byte(n >> 8) + buf[i+2] = byte(n >> 16) + } + } + return buf, nil +} + +type vmFilter struct { + execCount uint32 + global []byte + static []byte + code []command +} + +// execute implements v3filter type for VM based RAR 3 filters. +func (f *vmFilter) execute(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) { + if len(buf) > vmGlobalAddr { + return buf, errInvalidFilter + } + v := newVM(buf) + + // register setup + v.r[3] = vmGlobalAddr + v.r[4] = uint32(len(buf)) + v.r[5] = f.execCount + for i, n := range r { + v.r[i] = n + } + + // vm global data memory block + vg := v.m[vmGlobalAddr : vmGlobalAddr+vmGlobalSize] + + // initialize fixed global memory + for i, n := range v.r[:vmRegs-1] { + binary.LittleEndian.PutUint32(vg[i*4:], n) + } + binary.LittleEndian.PutUint32(vg[0x1c:], uint32(len(buf))) + binary.LittleEndian.PutUint64(vg[0x24:], uint64(offset)) + binary.LittleEndian.PutUint32(vg[0x2c:], f.execCount) + + // registers + v.r[6] = uint32(offset) + + // copy program global memory + var n int + if len(f.global) > 0 { + n = copy(vg[vmFixedGlobalSize:], f.global) // use saved global instead + } else { + n = copy(vg[vmFixedGlobalSize:], global) + } + copy(vg[vmFixedGlobalSize+n:], f.static) + + v.execute(f.code) + + f.execCount++ + + // keep largest global buffer + if cap(global) > cap(f.global) { + f.global = global[:0] + } else if len(f.global) > 0 { + f.global = f.global[:0] + } + + // check for global data to be saved for next program execution + globalSize := binary.LittleEndian.Uint32(vg[0x30:]) + if globalSize > 0 { + if globalSize > vmGlobalSize-vmFixedGlobalSize { + globalSize = vmGlobalSize - vmFixedGlobalSize + } + if cap(f.global) < int(globalSize) { + f.global = make([]byte, globalSize) + } else { + f.global = f.global[:globalSize] + } + copy(f.global, vg[vmFixedGlobalSize:]) + } + + // find program output + length := binary.LittleEndian.Uint32(vg[0x1c:]) & vmMask + start := binary.LittleEndian.Uint32(vg[0x20:]) & vmMask + if start+length > vmSize { + // TODO: error + start = 0 + length = 0 + } + if start != 0 && cap(v.m) > cap(buf) { + // Initial buffer was to small for vm. + // Copy output to beginning of vm memory so that decodeReader + // will re-use the newly allocated vm memory and we will not + // have to reallocate again next time. + copy(v.m, v.m[start:start+length]) + start = 0 + } + return v.m[start : start+length], nil +} + +// getV3Filter returns a V3 filter function from a code byte slice. +func getV3Filter(code []byte) (v3Filter, error) { + // check if filter is a known standard filter + c := crc32.ChecksumIEEE(code) + for _, f := range standardV3Filters { + if f.crc == c && f.len == len(code) { + return f.f, nil + } + } + + // create new vm filter + f := new(vmFilter) + r := newRarBitReader(bytes.NewReader(code[1:])) // skip first xor byte check + + // read static data + n, err := r.readBits(1) + if err != nil { + return nil, err + } + if n > 0 { + m, err := r.readUint32() + if err != nil { + return nil, err + } + f.static = make([]byte, m+1) + err = r.readFull(f.static) + if err != nil { + return nil, err + } + } + + f.code, err = readCommands(r) + if err == io.EOF { + err = nil + } + + return f.execute, err +} diff --git a/vendor/github.com/nwaples/rardecode/huffman.go b/vendor/github.com/nwaples/rardecode/huffman.go new file mode 100644 index 0000000..eb289b4 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/huffman.go @@ -0,0 +1,208 @@ +package rardecode + +import ( + "errors" + "io" +) + +const ( + maxCodeLength = 15 // maximum code length in bits + maxQuickBits = 10 + maxQuickSize = 1 << maxQuickBits +) + +var ( + errHuffDecodeFailed = errors.New("rardecode: huffman decode failed") + errInvalidLengthTable = errors.New("rardecode: invalid huffman code length table") +) + +type huffmanDecoder struct { + limit [maxCodeLength + 1]int + pos [maxCodeLength + 1]int + symbol []int + min uint + quickbits uint + quicklen [maxQuickSize]uint + quicksym [maxQuickSize]int +} + +func (h *huffmanDecoder) init(codeLengths []byte) { + var count [maxCodeLength + 1]int + + for _, n := range codeLengths { + if n == 0 { + continue + } + count[n]++ + } + + h.pos[0] = 0 + h.limit[0] = 0 + h.min = 0 + for i := uint(1); i <= maxCodeLength; i++ { + h.limit[i] = h.limit[i-1] + count[i]<<(maxCodeLength-i) + h.pos[i] = h.pos[i-1] + count[i-1] + if h.min == 0 && h.limit[i] > 0 { + h.min = i + } + } + + if cap(h.symbol) >= len(codeLengths) { + h.symbol = h.symbol[:len(codeLengths)] + for i := range h.symbol { + h.symbol[i] = 0 + } + } else { + h.symbol = make([]int, len(codeLengths)) + } + + copy(count[:], h.pos[:]) + for i, n := range codeLengths { + if n != 0 { + h.symbol[count[n]] = i + count[n]++ + } + } + + if len(codeLengths) >= 298 { + h.quickbits = maxQuickBits + } else { + h.quickbits = maxQuickBits - 3 + } + + bits := uint(1) + for i := 0; i < 1<= h.limit[bits] && bits < maxCodeLength { + bits++ + } + h.quicklen[i] = bits + + dist := v - h.limit[bits-1] + dist >>= (maxCodeLength - bits) + + pos := h.pos[bits] + dist + if pos < len(h.symbol) { + h.quicksym[i] = h.symbol[pos] + } else { + h.quicksym[i] = 0 + } + } +} + +func (h *huffmanDecoder) readSym(r bitReader) (int, error) { + bits := uint(maxCodeLength) + v, err := r.readBits(maxCodeLength) + if err != nil { + if err != io.EOF { + return 0, err + } + // fall back to 1 bit at a time if we read past EOF + for i := uint(1); i <= maxCodeLength; i++ { + b, err := r.readBits(1) + if err != nil { + return 0, err // not enough bits return error + } + v |= b << (maxCodeLength - i) + if v < h.limit[i] { + bits = i + break + } + } + } else { + if v < h.limit[h.quickbits] { + i := v >> (maxCodeLength - h.quickbits) + r.unreadBits(maxCodeLength - h.quicklen[i]) + return h.quicksym[i], nil + } + + for i, n := range h.limit[h.min:] { + if v < n { + bits = h.min + uint(i) + r.unreadBits(maxCodeLength - bits) + break + } + } + } + + dist := v - h.limit[bits-1] + dist >>= maxCodeLength - bits + + pos := h.pos[bits] + dist + if pos > len(h.symbol) { + return 0, errHuffDecodeFailed + } + + return h.symbol[pos], nil +} + +// readCodeLengthTable reads a new code length table into codeLength from br. +// If addOld is set the old table is added to the new one. +func readCodeLengthTable(br bitReader, codeLength []byte, addOld bool) error { + var bitlength [20]byte + for i := 0; i < len(bitlength); i++ { + n, err := br.readBits(4) + if err != nil { + return err + } + if n == 0xf { + cnt, err := br.readBits(4) + if err != nil { + return err + } + if cnt > 0 { + // array already zero'd dont need to explicitly set + i += cnt + 1 + continue + } + } + bitlength[i] = byte(n) + } + + var bl huffmanDecoder + bl.init(bitlength[:]) + + for i := 0; i < len(codeLength); i++ { + l, err := bl.readSym(br) + if err != nil { + return err + } + + if l < 16 { + if addOld { + codeLength[i] = (codeLength[i] + byte(l)) & 0xf + } else { + codeLength[i] = byte(l) + } + continue + } + + var count int + var value byte + + switch l { + case 16, 18: + count, err = br.readBits(3) + count += 3 + default: + count, err = br.readBits(7) + count += 11 + } + if err != nil { + return err + } + if l < 18 { + if i == 0 { + return errInvalidLengthTable + } + value = codeLength[i-1] + } + for ; count > 0 && i < len(codeLength); i++ { + codeLength[i] = value + count-- + } + i-- + } + return nil +} diff --git a/vendor/github.com/nwaples/rardecode/ppm_model.go b/vendor/github.com/nwaples/rardecode/ppm_model.go new file mode 100644 index 0000000..58a545a --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/ppm_model.go @@ -0,0 +1,1096 @@ +package rardecode + +import ( + "errors" + "io" +) + +const ( + rangeBottom = 1 << 15 + rangeTop = 1 << 24 + + maxFreq = 124 + + intBits = 7 + periodBits = 7 + binScale = 1 << (intBits + periodBits) + + n0 = 1 + n1 = 4 + n2 = 4 + n3 = 4 + n4 = (128 + 3 - 1*n1 - 2*n2 - 3*n3) / 4 + nIndexes = n0 + n1 + n2 + n3 + n4 + + // memory is allocated in units. A unit contains unitSize number of bytes. + // A unit can store one context or two states. + unitSize = 12 + + maxUint16 = 1<<16 - 1 + freeMark = -1 +) + +var ( + errCorruptPPM = errors.New("rardecode: corrupt ppm data") + + expEscape = []byte{25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2} + initBinEsc = []uint16{0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051} + + ns2Index [256]byte + ns2BSIndex [256]byte + + // units2Index maps the number of units in a block to a freelist index + units2Index [128 + 1]byte + // index2Units maps a freelist index to the size of the block in units + index2Units [nIndexes]int32 +) + +func init() { + ns2BSIndex[0] = 2 * 0 + ns2BSIndex[1] = 2 * 1 + for i := 2; i < 11; i++ { + ns2BSIndex[i] = 2 * 2 + } + for i := 11; i < 256; i++ { + ns2BSIndex[i] = 2 * 3 + } + + var j, n byte + for i := range ns2Index { + ns2Index[i] = n + if j <= 3 { + n++ + j = n + } else { + j-- + } + } + + var ii byte + var iu, units int32 + for i, n := range []int{n0, n1, n2, n3, n4} { + for j := 0; j < n; j++ { + units += int32(i) + index2Units[ii] = units + for iu <= units { + units2Index[iu] = ii + iu++ + } + ii++ + } + } +} + +type rangeCoder struct { + br io.ByteReader + code uint32 + low uint32 + rnge uint32 +} + +func (r *rangeCoder) init(br io.ByteReader) error { + r.br = br + r.low = 0 + r.rnge = ^uint32(0) + for i := 0; i < 4; i++ { + c, err := r.br.ReadByte() + if err != nil { + return err + } + r.code = r.code<<8 | uint32(c) + } + return nil +} + +func (r *rangeCoder) currentCount(scale uint32) uint32 { + r.rnge /= scale + return (r.code - r.low) / r.rnge +} + +func (r *rangeCoder) normalize() error { + for { + if r.low^(r.low+r.rnge) >= rangeTop { + if r.rnge >= rangeBottom { + return nil + } + r.rnge = -r.low & (rangeBottom - 1) + } + c, err := r.br.ReadByte() + if err != nil { + return err + } + r.code = r.code<<8 | uint32(c) + r.rnge <<= 8 + r.low <<= 8 + } +} + +func (r *rangeCoder) decode(lowCount, highCount uint32) error { + r.low += r.rnge * lowCount + r.rnge *= highCount - lowCount + + return r.normalize() +} + +type see2Context struct { + summ uint16 + shift byte + count byte +} + +func newSee2Context(i uint16) see2Context { + return see2Context{i << (periodBits - 4), (periodBits - 4), 4} +} + +func (s *see2Context) mean() uint32 { + if s == nil { + return 1 + } + n := s.summ >> s.shift + if n == 0 { + return 1 + } + s.summ -= n + return uint32(n) +} + +func (s *see2Context) update() { + if s == nil || s.shift >= periodBits { + return + } + s.count-- + if s.count == 0 { + s.summ += s.summ + s.count = 3 << s.shift + s.shift++ + } +} + +type state struct { + sym byte + freq byte + + // succ can point to a context or byte in memory. + // A context pointer is a positive integer. It is an index into the states + // array that points to the first of two states which the context is + // marshalled into. + // A byte pointer is a negative integer. The magnitude represents the position + // in bytes from the bottom of the memory. As memory is modelled as an array of + // states, this is used to calculate which state, and where in the state the + // byte is stored. + // A zero value represents a nil pointer. + succ int32 +} + +// uint16 return a uint16 stored in the sym and freq fields of a state +func (s state) uint16() uint16 { return uint16(s.sym) | uint16(s.freq)<<8 } + +// setUint16 stores a uint16 in the sym and freq fields of a state +func (s *state) setUint16(n uint16) { s.sym = byte(n); s.freq = byte(n >> 8) } + +// A context is marshalled into a slice of two states. +// The first state contains the number of states, and the suffix pointer. +// If there is only one state, the second state contains that state. +// If there is more than one state, the second state contains the summFreq +// and the index to the slice of states. +type context struct { + i int32 // index into the states array for context + s []state // slice of two states representing context + a *subAllocator +} + +// succPtr returns a pointer value for the context to be stored in a state.succ +func (c *context) succPtr() int32 { return c.i } + +func (c *context) numStates() int { return int(c.s[0].uint16()) } + +func (c *context) setNumStates(n int) { c.s[0].setUint16(uint16(n)) } + +func (c *context) statesIndex() int32 { return c.s[1].succ } + +func (c *context) setStatesIndex(n int32) { c.s[1].succ = n } + +func (c *context) suffix() *context { return c.a.succContext(c.s[0].succ) } + +func (c *context) setSuffix(sc *context) { c.s[0].succ = sc.i } + +func (c *context) summFreq() uint16 { return c.s[1].uint16() } + +func (c *context) setSummFreq(f uint16) { c.s[1].setUint16(f) } + +func (c *context) notEq(ctx *context) bool { return c.i != ctx.i } + +func (c *context) states() []state { + if ns := int32(c.s[0].uint16()); ns != 1 { + i := c.s[1].succ + return c.a.states[i : i+ns] + } + return c.s[1:] +} + +// shrinkStates shrinks the state list down to size states +func (c *context) shrinkStates(states []state, size int) []state { + i1 := units2Index[(len(states)+1)>>1] + i2 := units2Index[(size+1)>>1] + + if size == 1 { + // store state in context, and free states block + n := c.statesIndex() + c.s[1] = states[0] + states = c.s[1:] + c.a.addFreeBlock(n, i1) + } else if i1 != i2 { + if n := c.a.removeFreeBlock(i2); n > 0 { + // allocate new block and copy + copy(c.a.states[n:], states[:size]) + states = c.a.states[n:] + // free old block + c.a.addFreeBlock(c.statesIndex(), i1) + c.setStatesIndex(n) + } else { + // split current block, and free units not needed + n = c.statesIndex() + index2Units[i2]<<1 + u := index2Units[i1] - index2Units[i2] + c.a.freeUnits(n, u) + } + } + c.setNumStates(size) + return states[:size] +} + +// expandStates expands the states list by one +func (c *context) expandStates() []state { + states := c.states() + ns := len(states) + if ns == 1 { + s := states[0] + n := c.a.allocUnits(1) + if n == 0 { + return nil + } + c.setStatesIndex(n) + states = c.a.states[n:] + states[0] = s + } else if ns&0x1 == 0 { + u := ns >> 1 + i1 := units2Index[u] + i2 := units2Index[u+1] + if i1 != i2 { + n := c.a.allocUnits(i2) + if n == 0 { + return nil + } + copy(c.a.states[n:], states) + c.a.addFreeBlock(c.statesIndex(), i1) + c.setStatesIndex(n) + states = c.a.states[n:] + } + } + c.setNumStates(ns + 1) + return states[:ns+1] +} + +type subAllocator struct { + // memory for allocation is split into two heaps + + heap1MaxBytes int32 // maximum bytes available in heap1 + heap1Lo int32 // heap1 bottom in number of bytes + heap1Hi int32 // heap1 top in number of bytes + heap2Lo int32 // heap2 bottom index in states + heap2Hi int32 // heap2 top index in states + glueCount int + + // Each freeList entry contains an index into states for the beginning + // of a free block. The first state in that block may contain an index + // to another free block and so on. The size of the free block in units + // (2 states) for that freeList index can be determined from the + // index2Units array. + freeList [nIndexes]int32 + + // Instead of bytes, memory is represented by a slice of states. + // context's are marshalled to and from a pair of states. + // multiple bytes are stored in a state. + states []state +} + +func (a *subAllocator) init(maxMB int) { + bytes := int32(maxMB) << 20 + heap2Units := bytes / 8 / unitSize * 7 + a.heap1MaxBytes = bytes - heap2Units*unitSize + // Add one for the case when bytes are not a multiple of unitSize + heap1Units := a.heap1MaxBytes/unitSize + 1 + // Calculate total size in state's. Add 1 unit so we can reserve the first unit. + // This will allow us to use the zero index as a nil pointer. + n := int(1+heap1Units+heap2Units) * 2 + if cap(a.states) > n { + a.states = a.states[:n] + } else { + a.states = make([]state, n) + } +} + +func (a *subAllocator) restart() { + // Pad heap1 start by 1 unit and enough bytes so that there is no + // gap between heap1 end and heap2 start. + a.heap1Lo = unitSize + (unitSize - a.heap1MaxBytes%unitSize) + a.heap1Hi = unitSize + (a.heap1MaxBytes/unitSize+1)*unitSize + a.heap2Lo = a.heap1Hi / unitSize * 2 + a.heap2Hi = int32(len(a.states)) + a.glueCount = 0 + for i := range a.freeList { + a.freeList[i] = 0 + } + for i := range a.states { + a.states[i] = state{} + } +} + +// pushByte puts a byte on the heap and returns a state.succ index that +// can be used to retrieve it. +func (a *subAllocator) pushByte(c byte) int32 { + si := a.heap1Lo / 6 // state index + oi := a.heap1Lo % 6 // byte position in state + switch oi { + case 0: + a.states[si].sym = c + case 1: + a.states[si].freq = c + default: + n := (uint(oi) - 2) * 8 + mask := ^(uint32(0xFF) << n) + succ := uint32(a.states[si].succ) & mask + succ |= uint32(c) << n + a.states[si].succ = int32(succ) + } + a.heap1Lo++ + if a.heap1Lo >= a.heap1Hi { + return 0 + } + return -a.heap1Lo +} + +// popByte reverses the previous pushByte +func (a *subAllocator) popByte() { a.heap1Lo-- } + +// succByte returns a byte from the heap given a state.succ index +func (a *subAllocator) succByte(i int32) byte { + i = -i + si := i / 6 + oi := i % 6 + switch oi { + case 0: + return a.states[si].sym + case 1: + return a.states[si].freq + default: + n := (uint(oi) - 2) * 8 + succ := uint32(a.states[si].succ) >> n + return byte(succ & 0xff) + } +} + +// succContext returns a context given a state.succ index +func (a *subAllocator) succContext(i int32) *context { + if i <= 0 { + return nil + } + return &context{i: i, s: a.states[i : i+2 : i+2], a: a} +} + +// succIsNil returns whether a state.succ points to nothing +func (a *subAllocator) succIsNil(i int32) bool { return i == 0 } + +// nextByteAddr takes a state.succ value representing a pointer +// to a byte, and returns the next bytes address +func (a *subAllocator) nextByteAddr(n int32) int32 { return n - 1 } + +func (a *subAllocator) removeFreeBlock(i byte) int32 { + n := a.freeList[i] + if n != 0 { + a.freeList[i] = a.states[n].succ + a.states[n] = state{} + } + return n +} + +func (a *subAllocator) addFreeBlock(n int32, i byte) { + a.states[n].succ = a.freeList[i] + a.freeList[i] = n +} + +func (a *subAllocator) freeUnits(n, u int32) { + i := units2Index[u] + if u != index2Units[i] { + i-- + a.addFreeBlock(n, i) + u -= index2Units[i] + n += index2Units[i] << 1 + i = units2Index[u] + } + a.addFreeBlock(n, i) +} + +func (a *subAllocator) glueFreeBlocks() { + var freeIndex int32 + + for i, n := range a.freeList { + s := state{succ: freeMark} + s.setUint16(uint16(index2Units[i])) + for n != 0 { + states := a.states[n:] + states[1].succ = freeIndex + freeIndex = n + n = states[0].succ + states[0] = s + } + a.freeList[i] = 0 + } + + for i := freeIndex; i != 0; i = a.states[i+1].succ { + if a.states[i].succ != freeMark { + continue + } + u := int32(a.states[i].uint16()) + states := a.states[i+u<<1:] + for len(states) > 0 && states[0].succ == freeMark { + u += int32(states[0].uint16()) + if u > maxUint16 { + break + } + states[0].succ = 0 + a.states[i].setUint16(uint16(u)) + states = a.states[i+u<<1:] + } + } + + for n := freeIndex; n != 0; n = a.states[n+1].succ { + if a.states[n].succ != freeMark { + continue + } + a.states[n].succ = 0 + u := int32(a.states[n].uint16()) + m := n + for u > 128 { + a.addFreeBlock(m, nIndexes-1) + u -= 128 + m += 256 + } + a.freeUnits(m, u) + } +} + +func (a *subAllocator) allocUnitsRare(index byte) int32 { + if a.glueCount == 0 { + a.glueCount = 255 + a.glueFreeBlocks() + if n := a.removeFreeBlock(index); n > 0 { + return n + } + } + // try to find a larger free block and split it + for i := index + 1; i < nIndexes; i++ { + if n := a.removeFreeBlock(i); n > 0 { + u := index2Units[i] - index2Units[index] + a.freeUnits(n+index2Units[index]<<1, u) + return n + } + } + a.glueCount-- + + // try to allocate units from the top of heap1 + n := a.heap1Hi - index2Units[index]*unitSize + if n > a.heap1Lo { + a.heap1Hi = n + return a.heap1Hi / unitSize * 2 + } + return 0 +} + +func (a *subAllocator) allocUnits(i byte) int32 { + // try to allocate a free block + if n := a.removeFreeBlock(i); n > 0 { + return n + } + // try to allocate from the bottom of heap2 + n := index2Units[i] << 1 + if a.heap2Lo+n <= a.heap2Hi { + lo := a.heap2Lo + a.heap2Lo += n + return lo + } + return a.allocUnitsRare(i) +} + +func (a *subAllocator) newContext(s state, suffix *context) *context { + var n int32 + if a.heap2Lo < a.heap2Hi { + // allocate from top of heap2 + a.heap2Hi -= 2 + n = a.heap2Hi + } else if n = a.removeFreeBlock(1); n == 0 { + if n = a.allocUnitsRare(1); n == 0 { + return nil + } + } + c := &context{i: n, s: a.states[n : n+2 : n+2], a: a} + c.s[0] = state{} + c.setNumStates(1) + c.s[1] = s + if suffix != nil { + c.setSuffix(suffix) + } + return c +} + +func (a *subAllocator) newContextSize(ns int) *context { + c := a.newContext(state{}, nil) + c.setNumStates(ns) + i := units2Index[(ns+1)>>1] + n := a.allocUnits(i) + c.setStatesIndex(n) + return c +} + +type model struct { + maxOrder int + orderFall int + initRL int + runLength int + prevSuccess byte + escCount byte + prevSym byte + initEsc byte + minC *context + maxC *context + rc rangeCoder + a subAllocator + charMask [256]byte + binSumm [128][64]uint16 + see2Cont [25][16]see2Context +} + +func (m *model) restart() { + for i := range m.charMask { + m.charMask[i] = 0 + } + m.escCount = 1 + + if m.maxOrder < 12 { + m.initRL = -m.maxOrder - 1 + } else { + m.initRL = -12 - 1 + } + m.orderFall = m.maxOrder + m.runLength = m.initRL + m.prevSuccess = 0 + + m.a.restart() + + c := m.a.newContextSize(256) + c.setSummFreq(257) + states := c.states() + for i := range states { + states[i] = state{sym: byte(i), freq: 1} + } + m.minC = c + m.maxC = c + m.prevSym = 0 + + for i := range m.binSumm { + for j, esc := range initBinEsc { + n := binScale - esc/(uint16(i)+2) + for k := j; k < len(m.binSumm[i]); k += len(initBinEsc) { + m.binSumm[i][k] = n + } + } + } + + for i := range m.see2Cont { + see := newSee2Context(5*uint16(i) + 10) + for j := range m.see2Cont[i] { + m.see2Cont[i][j] = see + } + } +} + +func (m *model) init(br io.ByteReader, reset bool, maxOrder, maxMB int) error { + err := m.rc.init(br) + if err != nil { + return err + } + if !reset { + if m.minC == nil { + return errCorruptPPM + } + return nil + } + + m.a.init(maxMB) + + if maxOrder == 1 { + return errCorruptPPM + } + m.maxOrder = maxOrder + m.restart() + return nil +} + +func (m *model) rescale(s *state) *state { + if s.freq <= maxFreq { + return s + } + c := m.minC + + var summFreq uint16 + + s.freq += 4 + states := c.states() + escFreq := c.summFreq() + 4 + + for i := range states { + f := states[i].freq + escFreq -= uint16(f) + if m.orderFall != 0 { + f++ + } + f >>= 1 + summFreq += uint16(f) + states[i].freq = f + + if i == 0 || f <= states[i-1].freq { + continue + } + j := i - 1 + for j > 0 && f > states[j-1].freq { + j-- + } + t := states[i] + copy(states[j+1:i+1], states[j:i]) + states[j] = t + } + + i := len(states) - 1 + for states[i].freq == 0 { + i-- + escFreq++ + } + if i != len(states)-1 { + states = c.shrinkStates(states, i+1) + } + s = &states[0] + if i == 0 { + for { + s.freq -= s.freq >> 1 + escFreq >>= 1 + if escFreq <= 1 { + return s + } + } + } + summFreq += escFreq - (escFreq >> 1) + c.setSummFreq(summFreq) + return s +} + +func (m *model) decodeBinSymbol() (*state, error) { + c := m.minC + s := &c.states()[0] + + ns := c.suffix().numStates() + i := m.prevSuccess + ns2BSIndex[ns-1] + byte(m.runLength>>26)&0x20 + if m.prevSym >= 64 { + i += 8 + } + if s.sym >= 64 { + i += 2 * 8 + } + bs := &m.binSumm[s.freq-1][i] + mean := (*bs + 1<<(periodBits-2)) >> periodBits + + if m.rc.currentCount(binScale) < uint32(*bs) { + err := m.rc.decode(0, uint32(*bs)) + if s.freq < 128 { + s.freq++ + } + *bs += 1<>10] + m.charMask[s.sym] = m.escCount + m.prevSuccess = 0 + return nil, err +} + +func (m *model) decodeSymbol1() (*state, error) { + c := m.minC + states := c.states() + scale := uint32(c.summFreq()) + // protect against divide by zero + // TODO: look at why this happens, may be problem elsewhere + if scale == 0 { + return nil, errCorruptPPM + } + count := m.rc.currentCount(scale) + m.prevSuccess = 0 + + var n uint32 + for i := range states { + s := &states[i] + n += uint32(s.freq) + if n <= count { + continue + } + err := m.rc.decode(n-uint32(s.freq), n) + s.freq += 4 + c.setSummFreq(uint16(scale + 4)) + if i == 0 { + if 2*n > scale { + m.prevSuccess = 1 + m.runLength++ + } + } else { + if s.freq <= states[i-1].freq { + return s, err + } + states[i-1], states[i] = states[i], states[i-1] + s = &states[i-1] + } + return m.rescale(s), err + } + + for _, s := range states { + m.charMask[s.sym] = m.escCount + } + return nil, m.rc.decode(n, scale) +} + +func (m *model) makeEscFreq(c *context, numMasked int) *see2Context { + ns := c.numStates() + if ns == 256 { + return nil + } + diff := ns - numMasked + + var i int + if m.prevSym >= 64 { + i = 8 + } + if diff < c.suffix().numStates()-ns { + i++ + } + if int(c.summFreq()) < 11*ns { + i += 2 + } + if numMasked > diff { + i += 4 + } + return &m.see2Cont[ns2Index[diff-1]][i] +} + +func (m *model) decodeSymbol2(numMasked int) (*state, error) { + c := m.minC + + see := m.makeEscFreq(c, numMasked) + scale := see.mean() + + var i int + var hi uint32 + states := c.states() + sl := make([]*state, len(states)-numMasked) + for j := range sl { + for m.charMask[states[i].sym] == m.escCount { + i++ + } + hi += uint32(states[i].freq) + sl[j] = &states[i] + i++ + } + + scale += hi + count := m.rc.currentCount(scale) + + if count >= scale { + return nil, errCorruptPPM + } + if count >= hi { + err := m.rc.decode(hi, scale) + if see != nil { + see.summ += uint16(scale) + } + for _, s := range sl { + m.charMask[s.sym] = m.escCount + } + return nil, err + } + + hi = uint32(sl[0].freq) + for hi <= count { + sl = sl[1:] + hi += uint32(sl[0].freq) + } + s := sl[0] + + err := m.rc.decode(hi-uint32(s.freq), hi) + + see.update() + + m.escCount++ + m.runLength = m.initRL + + s.freq += 4 + c.setSummFreq(c.summFreq() + 4) + return m.rescale(s), err +} + +func (c *context) findState(sym byte) *state { + var i int + states := c.states() + for i = range states { + if states[i].sym == sym { + break + } + } + return &states[i] +} + +func (m *model) createSuccessors(s, ss *state) *context { + var sl []*state + + if m.orderFall != 0 { + sl = append(sl, s) + } + + c := m.minC + for suff := c.suffix(); suff != nil; suff = c.suffix() { + c = suff + + if ss == nil { + ss = c.findState(s.sym) + } + if ss.succ != s.succ { + c = m.a.succContext(ss.succ) + break + } + sl = append(sl, ss) + ss = nil + } + + if len(sl) == 0 { + return c + } + + var up state + up.sym = m.a.succByte(s.succ) + up.succ = m.a.nextByteAddr(s.succ) + + states := c.states() + if len(states) > 1 { + s = c.findState(up.sym) + + cf := uint16(s.freq) - 1 + s0 := c.summFreq() - uint16(len(states)) - cf + + if 2*cf <= s0 { + if 5*cf > s0 { + up.freq = 2 + } else { + up.freq = 1 + } + } else { + up.freq = byte(1 + (2*cf+3*s0-1)/(2*s0)) + } + } else { + up.freq = states[0].freq + } + + for i := len(sl) - 1; i >= 0; i-- { + c = m.a.newContext(up, c) + if c == nil { + return nil + } + sl[i].succ = c.succPtr() + } + return c +} + +func (m *model) update(s *state) { + if m.orderFall == 0 { + if c := m.a.succContext(s.succ); c != nil { + m.minC = c + m.maxC = c + return + } + } + + if m.escCount == 0 { + m.escCount = 1 + for i := range m.charMask { + m.charMask[i] = 0 + } + } + + var ss *state // matching minC.suffix state + + if s.freq < maxFreq/4 && m.minC.suffix() != nil { + c := m.minC.suffix() + states := c.states() + + var i int + if len(states) > 1 { + for states[i].sym != s.sym { + i++ + } + if i > 0 && states[i].freq >= states[i-1].freq { + states[i-1], states[i] = states[i], states[i-1] + i-- + } + if states[i].freq < maxFreq-9 { + states[i].freq += 2 + c.setSummFreq(c.summFreq() + 2) + } + } else if states[0].freq < 32 { + states[0].freq++ + } + ss = &states[i] // save later for createSuccessors + } + + if m.orderFall == 0 { + c := m.createSuccessors(s, ss) + if c == nil { + m.restart() + } else { + m.minC = c + m.maxC = c + s.succ = c.succPtr() + } + return + } + + succ := m.a.pushByte(s.sym) + if m.a.succIsNil(succ) { + m.restart() + return + } + + var minC *context + if m.a.succIsNil(s.succ) { + s.succ = succ + minC = m.minC + } else { + minC = m.a.succContext(s.succ) + if minC == nil { + minC = m.createSuccessors(s, ss) + if minC == nil { + m.restart() + return + } + } + m.orderFall-- + if m.orderFall == 0 { + succ = minC.succPtr() + if m.maxC.notEq(m.minC) { + m.a.popByte() + } + } + } + + n := m.minC.numStates() + s0 := int(m.minC.summFreq()) - n - int(s.freq-1) + for c := m.maxC; c.notEq(m.minC); c = c.suffix() { + var summFreq uint16 + + states := c.expandStates() + if states == nil { + m.restart() + return + } + if ns := len(states) - 1; ns != 1 { + summFreq = c.summFreq() + if 4*ns <= n && int(summFreq) <= 8*ns { + summFreq += 2 + } + if 2*ns < n { + summFreq++ + } + } else { + p := &states[0] + if p.freq < maxFreq/4-1 { + p.freq += p.freq + } else { + p.freq = maxFreq - 4 + } + summFreq = uint16(p.freq) + uint16(m.initEsc) + if n > 3 { + summFreq++ + } + } + + cf := 2 * int(s.freq) * int(summFreq+6) + sf := s0 + int(summFreq) + var freq byte + if cf >= 6*sf { + switch { + case cf >= 15*sf: + freq = 7 + case cf >= 12*sf: + freq = 6 + case cf >= 9*sf: + freq = 5 + default: + freq = 4 + } + summFreq += uint16(freq) + } else { + switch { + case cf >= 4*sf: + freq = 3 + case cf > sf: + freq = 2 + default: + freq = 1 + } + summFreq += 3 + } + states[len(states)-1] = state{sym: s.sym, freq: freq, succ: succ} + c.setSummFreq(summFreq) + } + m.minC = minC + m.maxC = minC +} + +func (m *model) ReadByte() (byte, error) { + if m.minC == nil { + return 0, errCorruptPPM + } + var s *state + var err error + if m.minC.numStates() == 1 { + s, err = m.decodeBinSymbol() + } else { + s, err = m.decodeSymbol1() + } + for s == nil && err == nil { + n := m.minC.numStates() + for m.minC.numStates() == n { + m.orderFall++ + m.minC = m.minC.suffix() + if m.minC == nil { + return 0, errCorruptPPM + } + } + s, err = m.decodeSymbol2(n) + } + if err != nil { + return 0, err + } + + // save sym so it doesn't get overwritten by a possible restart() + sym := s.sym + m.update(s) + m.prevSym = sym + return sym, nil +} diff --git a/vendor/github.com/nwaples/rardecode/reader.go b/vendor/github.com/nwaples/rardecode/reader.go new file mode 100644 index 0000000..03e88a8 --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/reader.go @@ -0,0 +1,369 @@ +package rardecode + +import ( + "bufio" + "bytes" + "errors" + "io" + "io/ioutil" + "os" + "time" +) + +// FileHeader HostOS types +const ( + HostOSUnknown = 0 + HostOSMSDOS = 1 + HostOSOS2 = 2 + HostOSWindows = 3 + HostOSUnix = 4 + HostOSMacOS = 5 + HostOSBeOS = 6 +) + +const ( + maxPassword = 128 +) + +var ( + errShortFile = errors.New("rardecode: decoded file too short") + errInvalidFileBlock = errors.New("rardecode: invalid file block") + errUnexpectedArcEnd = errors.New("rardecode: unexpected end of archive") + errBadFileChecksum = errors.New("rardecode: bad file checksum") +) + +type byteReader interface { + io.Reader + io.ByteReader +} + +type limitedReader struct { + r io.Reader + n int64 // bytes remaining + shortErr error // error returned when r returns io.EOF with n > 0 +} + +func (l *limitedReader) Read(p []byte) (int, error) { + if l.n <= 0 { + return 0, io.EOF + } + if int64(len(p)) > l.n { + p = p[0:l.n] + } + n, err := l.r.Read(p) + l.n -= int64(n) + if err == io.EOF && l.n > 0 { + return n, l.shortErr + } + return n, err +} + +type limitedByteReader struct { + limitedReader + br io.ByteReader +} + +func (l *limitedByteReader) ReadByte() (byte, error) { + if l.n <= 0 { + return 0, io.EOF + } + c, err := l.br.ReadByte() + if err == nil { + l.n-- + } else if err == io.EOF && l.n > 0 { + return 0, l.shortErr + } + return c, err +} + +// limitByteReader returns a limitedByteReader that reads from r and stops with +// io.EOF after n bytes. +// If r returns an io.EOF before reading n bytes, io.ErrUnexpectedEOF is returned. +func limitByteReader(r byteReader, n int64) *limitedByteReader { + return &limitedByteReader{limitedReader{r, n, io.ErrUnexpectedEOF}, r} +} + +// fileChecksum allows file checksum validations to be performed. +// File contents must first be written to fileChecksum. Then valid is +// called to perform the file checksum calculation to determine +// if the file contents are valid or not. +type fileChecksum interface { + io.Writer + valid() bool +} + +// FileHeader represents a single file in a RAR archive. +type FileHeader struct { + Name string // file name using '/' as the directory separator + IsDir bool // is a directory + HostOS byte // Host OS the archive was created on + Attributes int64 // Host OS specific file attributes + PackedSize int64 // packed file size (or first block if the file spans volumes) + UnPackedSize int64 // unpacked file size + UnKnownSize bool // unpacked file size is not known + ModificationTime time.Time // modification time (non-zero if set) + CreationTime time.Time // creation time (non-zero if set) + AccessTime time.Time // access time (non-zero if set) + Version int // file version +} + +// Mode returns an os.FileMode for the file, calculated from the Attributes field. +func (f *FileHeader) Mode() os.FileMode { + var m os.FileMode + + if f.IsDir { + m = os.ModeDir + } + if f.HostOS == HostOSWindows { + if f.IsDir { + m |= 0777 + } else if f.Attributes&1 > 0 { + m |= 0444 // readonly + } else { + m |= 0666 + } + return m + } + // assume unix perms for all remaining os types + m |= os.FileMode(f.Attributes) & os.ModePerm + + // only check other bits on unix host created archives + if f.HostOS != HostOSUnix { + return m + } + + if f.Attributes&0x200 != 0 { + m |= os.ModeSticky + } + if f.Attributes&0x400 != 0 { + m |= os.ModeSetgid + } + if f.Attributes&0x800 != 0 { + m |= os.ModeSetuid + } + + // Check for additional file types. + if f.Attributes&0xF000 == 0xA000 { + m |= os.ModeSymlink + } + return m +} + +// fileBlockHeader represents a file block in a RAR archive. +// Files may comprise one or more file blocks. +// Solid files retain decode tables and dictionary from previous solid files in the archive. +type fileBlockHeader struct { + first bool // first block in file + last bool // last block in file + solid bool // file is solid + winSize uint // log base 2 of decode window size + cksum fileChecksum // file checksum + decoder decoder // decoder to use for file + key []byte // key for AES, non-empty if file encrypted + iv []byte // iv for AES, non-empty if file encrypted + FileHeader +} + +// fileBlockReader provides sequential access to file blocks in a RAR archive. +type fileBlockReader interface { + io.Reader // Read's read data from the current file block + io.ByteReader // Read bytes from current file block + next() (*fileBlockHeader, error) // reads the next file block header at current position + reset() // resets encryption + isSolid() bool // is archive solid + version() int // returns current archive format version +} + +// packedFileReader provides sequential access to packed files in a RAR archive. +type packedFileReader struct { + r fileBlockReader + h *fileBlockHeader // current file header +} + +// nextBlockInFile reads the next file block in the current file at the current +// archive file position, or returns an error if there is a problem. +// It is invalid to call this when already at the last block in the current file. +func (f *packedFileReader) nextBlockInFile() error { + h, err := f.r.next() + if err != nil { + if err == io.EOF { + // archive ended, but file hasn't + return errUnexpectedArcEnd + } + return err + } + if h.first || h.Name != f.h.Name { + return errInvalidFileBlock + } + f.h = h + return nil +} + +// next advances to the next packed file in the RAR archive. +func (f *packedFileReader) next() (*fileBlockHeader, error) { + if f.h != nil { + // skip to last block in current file + for !f.h.last { + // discard remaining block data + if _, err := io.Copy(ioutil.Discard, f.r); err != nil { + return nil, err + } + if err := f.nextBlockInFile(); err != nil { + return nil, err + } + } + // discard last block data + if _, err := io.Copy(ioutil.Discard, f.r); err != nil { + return nil, err + } + } + var err error + f.h, err = f.r.next() // get next file block + if err != nil { + if err == errArchiveEnd { + return nil, io.EOF + } + return nil, err + } + if !f.h.first { + return nil, errInvalidFileBlock + } + return f.h, nil +} + +// Read reads the packed data for the current file into p. +func (f *packedFileReader) Read(p []byte) (int, error) { + n, err := f.r.Read(p) // read current block data + for err == io.EOF { // current block empty + if n > 0 { + return n, nil + } + if f.h == nil || f.h.last { + return 0, io.EOF // last block so end of file + } + if err := f.nextBlockInFile(); err != nil { + return 0, err + } + n, err = f.r.Read(p) // read new block data + } + return n, err +} + +func (f *packedFileReader) ReadByte() (byte, error) { + c, err := f.r.ReadByte() // read current block data + for err == io.EOF && f.h != nil && !f.h.last { // current block empty + if err := f.nextBlockInFile(); err != nil { + return 0, err + } + c, err = f.r.ReadByte() // read new block data + } + return c, err +} + +// Reader provides sequential access to files in a RAR archive. +type Reader struct { + r io.Reader // reader for current unpacked file + pr packedFileReader // reader for current packed file + dr decodeReader // reader for decoding and filters if file is compressed + cksum fileChecksum // current file checksum + solidr io.Reader // reader for solid file +} + +// Read reads from the current file in the RAR archive. +func (r *Reader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + if err == io.EOF && r.cksum != nil && !r.cksum.valid() { + return n, errBadFileChecksum + } + return n, err +} + +// Next advances to the next file in the archive. +func (r *Reader) Next() (*FileHeader, error) { + if r.solidr != nil { + // solid files must be read fully to update decoder information + if _, err := io.Copy(ioutil.Discard, r.solidr); err != nil { + return nil, err + } + } + + h, err := r.pr.next() // skip to next file + if err != nil { + return nil, err + } + r.solidr = nil + + br := byteReader(&r.pr) // start with packed file reader + + // check for encryption + if len(h.key) > 0 && len(h.iv) > 0 { + br = newAesDecryptReader(br, h.key, h.iv) // decrypt + } + r.r = br + // check for compression + if h.decoder != nil { + err = r.dr.init(br, h.decoder, h.winSize, !h.solid) + if err != nil { + return nil, err + } + r.r = &r.dr + if r.pr.r.isSolid() { + r.solidr = r.r + } + } + if h.UnPackedSize >= 0 && !h.UnKnownSize { + // Limit reading to UnPackedSize as there may be padding + r.r = &limitedReader{r.r, h.UnPackedSize, errShortFile} + } + r.cksum = h.cksum + if r.cksum != nil { + r.r = io.TeeReader(r.r, h.cksum) // write file data to checksum as it is read + } + fh := new(FileHeader) + *fh = h.FileHeader + return fh, nil +} + +func (r *Reader) init(fbr fileBlockReader) { + r.r = bytes.NewReader(nil) // initial reads will always return EOF + r.pr.r = fbr +} + +// NewReader creates a Reader reading from r. +// NewReader only supports single volume archives. +// Multi-volume archives must use OpenReader. +func NewReader(r io.Reader, password string) (*Reader, error) { + br, ok := r.(*bufio.Reader) + if !ok { + br = bufio.NewReader(r) + } + fbr, err := newFileBlockReader(br, password) + if err != nil { + return nil, err + } + rr := new(Reader) + rr.init(fbr) + return rr, nil +} + +type ReadCloser struct { + v *volume + Reader +} + +// Close closes the rar file. +func (rc *ReadCloser) Close() error { + return rc.v.Close() +} + +// OpenReader opens a RAR archive specified by the name and returns a ReadCloser. +func OpenReader(name, password string) (*ReadCloser, error) { + v, err := openVolume(name, password) + if err != nil { + return nil, err + } + rc := new(ReadCloser) + rc.v = v + rc.Reader.init(v) + return rc, nil +} diff --git a/vendor/github.com/nwaples/rardecode/vm.go b/vendor/github.com/nwaples/rardecode/vm.go new file mode 100644 index 0000000..fd26a5a --- /dev/null +++ b/vendor/github.com/nwaples/rardecode/vm.go @@ -0,0 +1,687 @@ +package rardecode + +import ( + "encoding/binary" + "errors" +) + +const ( + // vm flag bits + flagC = 1 // Carry + flagZ = 2 // Zero + flagS = 0x80000000 // Sign + + maxCommands = 25000000 // maximum number of commands that can be run in a program + + vmRegs = 8 // number if registers + vmSize = 0x40000 // memory size + vmMask = vmSize - 1 +) + +var ( + errInvalidVMInstruction = errors.New("rardecode: invalid vm instruction") +) + +type vm struct { + ip uint32 // instruction pointer + ipMod bool // ip was modified + fl uint32 // flag bits + r [vmRegs]uint32 // registers + m []byte // memory +} + +func (v *vm) setIP(ip uint32) { + v.ip = ip + v.ipMod = true +} + +// execute runs a list of commands on the vm. +func (v *vm) execute(cmd []command) { + v.ip = 0 // reset instruction pointer + for n := 0; n < maxCommands; n++ { + ip := v.ip + if ip >= uint32(len(cmd)) { + return + } + ins := cmd[ip] + ins.f(v, ins.bm, ins.op) // run cpu instruction + if v.ipMod { + // command modified ip, don't increment + v.ipMod = false + } else { + v.ip++ // increment ip for next command + } + } +} + +// newVM creates a new RAR virtual machine using the byte slice as memory. +func newVM(mem []byte) *vm { + v := new(vm) + + if cap(mem) < vmSize+4 { + v.m = make([]byte, vmSize+4) + copy(v.m, mem) + } else { + v.m = mem[:vmSize+4] + for i := len(mem); i < len(v.m); i++ { + v.m[i] = 0 + } + } + v.r[7] = vmSize + return v +} + +type operand interface { + get(v *vm, byteMode bool) uint32 + set(v *vm, byteMode bool, n uint32) +} + +// Immediate Operand +type opI uint32 + +func (op opI) get(v *vm, bm bool) uint32 { return uint32(op) } +func (op opI) set(v *vm, bm bool, n uint32) {} + +// Direct Operand +type opD uint32 + +func (op opD) get(v *vm, byteMode bool) uint32 { + if byteMode { + return uint32(v.m[op]) + } + return binary.LittleEndian.Uint32(v.m[op:]) +} + +func (op opD) set(v *vm, byteMode bool, n uint32) { + if byteMode { + v.m[op] = byte(n) + } else { + binary.LittleEndian.PutUint32(v.m[op:], n) + } +} + +// Register Operand +type opR uint32 + +func (op opR) get(v *vm, byteMode bool) uint32 { + if byteMode { + return v.r[op] & 0xFF + } + return v.r[op] +} + +func (op opR) set(v *vm, byteMode bool, n uint32) { + if byteMode { + v.r[op] = (v.r[op] & 0xFFFFFF00) | (n & 0xFF) + } else { + v.r[op] = n + } +} + +// Register Indirect Operand +type opRI uint32 + +func (op opRI) get(v *vm, byteMode bool) uint32 { + i := v.r[op] & vmMask + if byteMode { + return uint32(v.m[i]) + } + return binary.LittleEndian.Uint32(v.m[i:]) +} +func (op opRI) set(v *vm, byteMode bool, n uint32) { + i := v.r[op] & vmMask + if byteMode { + v.m[i] = byte(n) + } else { + binary.LittleEndian.PutUint32(v.m[i:], n) + } +} + +// Base Plus Index Indirect Operand +type opBI struct { + r uint32 + i uint32 +} + +func (op opBI) get(v *vm, byteMode bool) uint32 { + i := (v.r[op.r] + op.i) & vmMask + if byteMode { + return uint32(v.m[i]) + } + return binary.LittleEndian.Uint32(v.m[i:]) +} +func (op opBI) set(v *vm, byteMode bool, n uint32) { + i := (v.r[op.r] + op.i) & vmMask + if byteMode { + v.m[i] = byte(n) + } else { + binary.LittleEndian.PutUint32(v.m[i:], n) + } +} + +type commandFunc func(v *vm, byteMode bool, op []operand) + +type command struct { + f commandFunc + bm bool // is byte mode + op []operand +} + +var ( + ops = []struct { + f commandFunc + byteMode bool // supports byte mode + nops int // number of operands + jop bool // is a jump op + }{ + {mov, true, 2, false}, + {cmp, true, 2, false}, + {add, true, 2, false}, + {sub, true, 2, false}, + {jz, false, 1, true}, + {jnz, false, 1, true}, + {inc, true, 1, false}, + {dec, true, 1, false}, + {jmp, false, 1, true}, + {xor, true, 2, false}, + {and, true, 2, false}, + {or, true, 2, false}, + {test, true, 2, false}, + {js, false, 1, true}, + {jns, false, 1, true}, + {jb, false, 1, true}, + {jbe, false, 1, true}, + {ja, false, 1, true}, + {jae, false, 1, true}, + {push, false, 1, false}, + {pop, false, 1, false}, + {call, false, 1, true}, + {ret, false, 0, false}, + {not, true, 1, false}, + {shl, true, 2, false}, + {shr, true, 2, false}, + {sar, true, 2, false}, + {neg, true, 1, false}, + {pusha, false, 0, false}, + {popa, false, 0, false}, + {pushf, false, 0, false}, + {popf, false, 0, false}, + {movzx, false, 2, false}, + {movsx, false, 2, false}, + {xchg, true, 2, false}, + {mul, true, 2, false}, + {div, true, 2, false}, + {adc, true, 2, false}, + {sbb, true, 2, false}, + {print, false, 0, false}, + } +) + +func mov(v *vm, bm bool, op []operand) { + op[0].set(v, bm, op[1].get(v, bm)) +} + +func cmp(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + r := v1 - op[1].get(v, bm) + if r == 0 { + v.fl = flagZ + } else { + v.fl = 0 + if r > v1 { + v.fl = flagC + } + v.fl |= r & flagS + } +} + +func add(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + r := v1 + op[1].get(v, bm) + v.fl = 0 + signBit := uint32(flagS) + if bm { + r &= 0xFF + signBit = 0x80 + } + if r < v1 { + v.fl |= flagC + } + if r == 0 { + v.fl |= flagZ + } else if r&signBit > 0 { + v.fl |= flagS + } + op[0].set(v, bm, r) +} + +func sub(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + r := v1 - op[1].get(v, bm) + v.fl = 0 + + if r == 0 { + v.fl = flagZ + } else { + v.fl = 0 + if r > v1 { + v.fl = flagC + } + v.fl |= r & flagS + } + op[0].set(v, bm, r) +} + +func jz(v *vm, bm bool, op []operand) { + if v.fl&flagZ > 0 { + v.setIP(op[0].get(v, false)) + } +} + +func jnz(v *vm, bm bool, op []operand) { + if v.fl&flagZ == 0 { + v.setIP(op[0].get(v, false)) + } +} + +func inc(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) + 1 + if bm { + r &= 0xFF + } + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } +} + +func dec(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) - 1 + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } +} + +func jmp(v *vm, bm bool, op []operand) { + v.setIP(op[0].get(v, false)) +} + +func xor(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) ^ op[1].get(v, bm) + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } +} + +func and(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) & op[1].get(v, bm) + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } +} + +func or(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) | op[1].get(v, bm) + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } +} + +func test(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) & op[1].get(v, bm) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } +} + +func js(v *vm, bm bool, op []operand) { + if v.fl&flagS > 0 { + v.setIP(op[0].get(v, false)) + } +} + +func jns(v *vm, bm bool, op []operand) { + if v.fl&flagS == 0 { + v.setIP(op[0].get(v, false)) + } +} + +func jb(v *vm, bm bool, op []operand) { + if v.fl&flagC > 0 { + v.setIP(op[0].get(v, false)) + } +} + +func jbe(v *vm, bm bool, op []operand) { + if v.fl&(flagC|flagZ) > 0 { + v.setIP(op[0].get(v, false)) + } +} + +func ja(v *vm, bm bool, op []operand) { + if v.fl&(flagC|flagZ) == 0 { + v.setIP(op[0].get(v, false)) + } +} + +func jae(v *vm, bm bool, op []operand) { + if v.fl&flagC == 0 { + v.setIP(op[0].get(v, false)) + } +} + +func push(v *vm, bm bool, op []operand) { + v.r[7] -= 4 + opRI(7).set(v, false, op[0].get(v, false)) + +} + +func pop(v *vm, bm bool, op []operand) { + op[0].set(v, false, opRI(7).get(v, false)) + v.r[7] += 4 +} + +func call(v *vm, bm bool, op []operand) { + v.r[7] -= 4 + opRI(7).set(v, false, v.ip+1) + v.setIP(op[0].get(v, false)) +} + +func ret(v *vm, bm bool, op []operand) { + r7 := v.r[7] + if r7 >= vmSize { + v.setIP(0xFFFFFFFF) // trigger end of program + } else { + v.setIP(binary.LittleEndian.Uint32(v.m[r7:])) + v.r[7] += 4 + } +} + +func not(v *vm, bm bool, op []operand) { + op[0].set(v, bm, ^op[0].get(v, bm)) +} + +func shl(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + v2 := op[1].get(v, bm) + r := v1 << v2 + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } + if (v1<<(v2-1))&0x80000000 > 0 { + v.fl |= flagC + } +} + +func shr(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + v2 := op[1].get(v, bm) + r := v1 >> v2 + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } + if (v1>>(v2-1))&0x1 > 0 { + v.fl |= flagC + } +} + +func sar(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + v2 := op[1].get(v, bm) + r := uint32(int32(v1) >> v2) + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } + if (v1>>(v2-1))&0x1 > 0 { + v.fl |= flagC + } +} + +func neg(v *vm, bm bool, op []operand) { + r := 0 - op[0].get(v, bm) + op[0].set(v, bm, r) + if r == 0 { + v.fl = flagZ + } else { + v.fl = r&flagS | flagC + } +} + +func pusha(v *vm, bm bool, op []operand) { + sp := opD(v.r[7]) + for _, r := range v.r { + sp = (sp - 4) & vmMask + sp.set(v, false, r) + } + v.r[7] = uint32(sp) +} + +func popa(v *vm, bm bool, op []operand) { + sp := opD(v.r[7]) + for i := 7; i >= 0; i-- { + v.r[i] = sp.get(v, false) + sp = (sp + 4) & vmMask + } +} + +func pushf(v *vm, bm bool, op []operand) { + v.r[7] -= 4 + opRI(7).set(v, false, v.fl) +} + +func popf(v *vm, bm bool, op []operand) { + v.fl = opRI(7).get(v, false) + v.r[7] += 4 +} + +func movzx(v *vm, bm bool, op []operand) { + op[0].set(v, false, op[1].get(v, true)) +} + +func movsx(v *vm, bm bool, op []operand) { + op[0].set(v, false, uint32(int8(op[1].get(v, true)))) +} + +func xchg(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + op[0].set(v, bm, op[1].get(v, bm)) + op[1].set(v, bm, v1) +} + +func mul(v *vm, bm bool, op []operand) { + r := op[0].get(v, bm) * op[1].get(v, bm) + op[0].set(v, bm, r) +} + +func div(v *vm, bm bool, op []operand) { + div := op[1].get(v, bm) + if div != 0 { + r := op[0].get(v, bm) / div + op[0].set(v, bm, r) + } +} + +func adc(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + fc := v.fl & flagC + r := v1 + op[1].get(v, bm) + fc + if bm { + r &= 0xFF + } + op[0].set(v, bm, r) + + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } + if r < v1 || (r == v1 && fc > 0) { + v.fl |= flagC + } +} + +func sbb(v *vm, bm bool, op []operand) { + v1 := op[0].get(v, bm) + fc := v.fl & flagC + r := v1 - op[1].get(v, bm) - fc + if bm { + r &= 0xFF + } + op[0].set(v, bm, r) + + if r == 0 { + v.fl = flagZ + } else { + v.fl = r & flagS + } + if r > v1 || (r == v1 && fc > 0) { + v.fl |= flagC + } +} + +func print(v *vm, bm bool, op []operand) { + // TODO: ignore print for the moment +} + +func decodeArg(br *rarBitReader, byteMode bool) (operand, error) { + n, err := br.readBits(1) + if err != nil { + return nil, err + } + if n > 0 { // Register + n, err = br.readBits(3) + return opR(n), err + } + n, err = br.readBits(1) + if err != nil { + return nil, err + } + if n == 0 { // Immediate + if byteMode { + n, err = br.readBits(8) + } else { + m, err := br.readUint32() + return opI(m), err + } + return opI(n), err + } + n, err = br.readBits(1) + if err != nil { + return nil, err + } + if n == 0 { + // Register Indirect + n, err = br.readBits(3) + return opRI(n), err + } + n, err = br.readBits(1) + if err != nil { + return nil, err + } + if n == 0 { + // Base + Index Indirect + n, err = br.readBits(3) + if err != nil { + return nil, err + } + i, err := br.readUint32() + return opBI{r: uint32(n), i: i}, err + } + // Direct addressing + m, err := br.readUint32() + return opD(m & vmMask), err +} + +func fixJumpOp(op operand, off int) operand { + n, ok := op.(opI) + if !ok { + return op + } + if n >= 256 { + return n - 256 + } + if n >= 136 { + n -= 264 + } else if n >= 16 { + n -= 8 + } else if n >= 8 { + n -= 16 + } + return n + opI(off) +} + +func readCommands(br *rarBitReader) ([]command, error) { + var cmds []command + + for { + code, err := br.readBits(4) + if err != nil { + return cmds, err + } + if code&0x08 > 0 { + n, err := br.readBits(2) + if err != nil { + return cmds, err + } + code = (code<<2 | n) - 24 + } + + if code >= len(ops) { + return cmds, errInvalidVMInstruction + } + ins := ops[code] + + var com command + + if ins.byteMode { + n, err := br.readBits(1) + if err != nil { + return cmds, err + } + com.bm = n > 0 + } + com.f = ins.f + + if ins.nops > 0 { + com.op = make([]operand, ins.nops) + com.op[0], err = decodeArg(br, com.bm) + if err != nil { + return cmds, err + } + if ins.nops == 2 { + com.op[1], err = decodeArg(br, com.bm) + if err != nil { + return cmds, err + } + } else if ins.jop { + com.op[0] = fixJumpOp(com.op[0], len(cmds)) + } + } + cmds = append(cmds, com) + } +} diff --git a/vendor/github.com/ulikunitz/xz/.gitignore b/vendor/github.com/ulikunitz/xz/.gitignore new file mode 100644 index 0000000..e3c2fc2 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/.gitignore @@ -0,0 +1,25 @@ +# .gitignore + +TODO.html +README.html + +lzma/writer.txt +lzma/reader.txt + +cmd/gxz/gxz +cmd/xb/xb + +# test executables +*.test + +# profile files +*.out + +# vim swap file +.*.swp + +# executables on windows +*.exe + +# default compression test file +enwik8* diff --git a/vendor/github.com/ulikunitz/xz/LICENSE b/vendor/github.com/ulikunitz/xz/LICENSE new file mode 100644 index 0000000..58ebdc1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2014-2016 Ulrich Kunitz +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* My name, Ulrich Kunitz, may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ulikunitz/xz/README.md b/vendor/github.com/ulikunitz/xz/README.md new file mode 100644 index 0000000..969ae7a --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/README.md @@ -0,0 +1,71 @@ +# Package xz + +This Go language package supports the reading and writing of xz +compressed streams. It includes also a gxz command for compressing and +decompressing data. The package is completely written in Go and doesn't +have any dependency on any C code. + +The package is currently under development. There might be bugs and APIs +are not considered stable. At this time the package cannot compete with +the xz tool regarding compression speed and size. The algorithms there +have been developed over a long time and are highly optimized. However +there are a number of improvements planned and I'm very optimistic about +parallel compression and decompression. Stay tuned! + +# Using the API + +The following example program shows how to use the API. + + package main + + import ( + "bytes" + "io" + "log" + "os" + + "github.com/ulikunitz/xz" + ) + + func main() { + const text = "The quick brown fox jumps over the lazy dog.\n" + var buf bytes.Buffer + // compress text + w, err := xz.NewWriter(&buf) + if err != nil { + log.Fatalf("xz.NewWriter error %s", err) + } + if _, err := io.WriteString(w, text); err != nil { + log.Fatalf("WriteString error %s", err) + } + if err := w.Close(); err != nil { + log.Fatalf("w.Close error %s", err) + } + // decompress buffer and write output to stdout + r, err := xz.NewReader(&buf) + if err != nil { + log.Fatalf("NewReader error %s", err) + } + if _, err = io.Copy(os.Stdout, r); err != nil { + log.Fatalf("io.Copy error %s", err) + } + } + +# Using the gxz compression tool + +The package includes a gxz command line utility for compression and +decompression. + +Use following command for installation: + + $ go get github.com/ulikunitz/xz/cmd/gxz + +To test it call the following command. + + $ gxz bigfile + +After some time a much smaller file bigfile.xz will replace bigfile. +To decompress it use the following command. + + $ gxz -d bigfile.xz + diff --git a/vendor/github.com/ulikunitz/xz/TODO.md b/vendor/github.com/ulikunitz/xz/TODO.md new file mode 100644 index 0000000..7b34c0c --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/TODO.md @@ -0,0 +1,315 @@ +# TODO list + +## Release v0.6 + +1. Review encoder and check for lzma improvements under xz. +2. Fix binary tree matcher. +3. Compare compression ratio with xz tool using comparable parameters + and optimize parameters +4. Do some optimizations + - rename operation action and make it a simple type of size 8 + - make maxMatches, wordSize parameters + - stop searching after a certain length is found (parameter sweetLen) + +## Release v0.7 + +1. Optimize code +2. Do statistical analysis to get linear presets. +3. Test sync.Pool compatability for xz and lzma Writer and Reader +3. Fuzz optimized code. + +## Release v0.8 + +1. Support parallel go routines for writing and reading xz files. +2. Support a ReaderAt interface for xz files with small block sizes. +3. Improve compatibility between gxz and xz +4. Provide manual page for gxz + +## Release v0.9 + +1. Improve documentation +2. Fuzz again + +## Release v1.0 + +1. Full functioning gxz +2. Add godoc URL to README.md (godoc.org) +3. Resolve all issues. +4. Define release candidates. +5. Public announcement. + +## Package lzma + +### Release v0.6 + +- Rewrite Encoder into a simple greedy one-op-at-a-time encoder + including + + simple scan at the dictionary head for the same byte + + use the killer byte (requiring matches to get longer, the first + test should be the byte that would make the match longer) + + +## Optimizations + +- There may be a lot of false sharing in lzma.State; check whether this + can be improved by reorganizing the internal structure of it. +- Check whether batching encoding and decoding improves speed. + +### DAG optimizations + +- Use full buffer to create minimal bit-length above range encoder. +- Might be too slow (see v0.4) + +### Different match finders + +- hashes with 2, 3 characters additional to 4 characters +- binary trees with 2-7 characters (uint64 as key, use uint32 as + pointers into a an array) +- rb-trees with 2-7 characters (uint64 as key, use uint32 as pointers + into an array with bit-steeling for the colors) + +## Release Procedure + +- execute goch -l for all packages; probably with lower param like 0.5. +- check orthography with gospell +- Write release notes in doc/relnotes. +- Update README.md +- xb copyright . in xz directory to ensure all new files have Copyright + header +- VERSION= go generate github.com/ulikunitz/xz/... to update + version files +- Execute test for Linux/amd64, Linux/x86 and Windows/amd64. +- Update TODO.md - write short log entry +- git checkout master && git merge dev +- git tag -a +- git push + +## Log + +### 2017-06-05 + +Release v0.5.4 fixes issues #15 of another problem with the padding size +check for the xz block header. I removed the check completely. + +### 2017-02-15 + +Release v0.5.3 fixes issue #12 regarding the decompression of an empty +XZ stream. Many thanks to Tomasz Kłak, who reported the issue. + +### 2016-12-02 + +Release v0.5.2 became necessary to allow the decoding of xz files with +4-byte padding in the block header. Many thanks to Greg, who reported +the issue. + +### 2016-07-23 + +Release v0.5.1 became necessary to fix problems with 32-bit platforms. +Many thanks to Bruno Brigas, who reported the issue. + +### 2016-07-04 + +Release v0.5 provides improvements to the compressor and provides support for +the decompression of xz files with multiple xz streams. + +### 2016-01-31 + +Another compression rate increase by checking the byte at length of the +best match first, before checking the whole prefix. This makes the +compressor even faster. We have now a large time budget to beat the +compression ratio of the xz tool. For enwik8 we have now over 40 seconds +to reduce the compressed file size for another 7 MiB. + +### 2016-01-30 + +I simplified the encoder. Speed and compression rate increased +dramatically. A high compression rate affects also the decompression +speed. The approach with the buffer and optimizing for operation +compression rate has not been successful. Going for the maximum length +appears to be the best approach. + +### 2016-01-28 + +The release v0.4 is ready. It provides a working xz implementation, +which is rather slow, but works and is interoperable with the xz tool. +It is an important milestone. + +### 2016-01-10 + +I have the first working implementation of an xz reader and writer. I'm +happy about reaching this milestone. + +### 2015-12-02 + +I'm now ready to implement xz because, I have a working LZMA2 +implementation. I decided today that v0.4 will use the slow encoder +using the operations buffer to be able to go back, if I intend to do so. + +### 2015-10-21 + +I have restarted the work on the library. While trying to implement +LZMA2, I discovered that I need to resimplify the encoder and decoder +functions. The option approach is too complicated. Using a limited byte +writer and not caring for written bytes at all and not to try to handle +uncompressed data simplifies the LZMA encoder and decoder much. +Processing uncompressed data and handling limits is a feature of the +LZMA2 format not of LZMA. + +I learned an interesting method from the LZO format. If the last copy is +too far away they are moving the head one 2 bytes and not 1 byte to +reduce processing times. + +### 2015-08-26 + +I have now reimplemented the lzma package. The code is reasonably fast, +but can still be optimized. The next step is to implement LZMA2 and then +xz. + +### 2015-07-05 + +Created release v0.3. The version is the foundation for a full xz +implementation that is the target of v0.4. + +### 2015-06-11 + +The gflag package has been developed because I couldn't use flag and +pflag for a fully compatible support of gzip's and lzma's options. It +seems to work now quite nicely. + +### 2015-06-05 + +The overflow issue was interesting to research, however Henry S. Warren +Jr. Hacker's Delight book was very helpful as usual and had the issue +explained perfectly. Fefe's information on his website was based on the +C FAQ and quite bad, because it didn't address the issue of -MININT == +MININT. + +### 2015-06-04 + +It has been a productive day. I improved the interface of lzma.Reader +and lzma.Writer and fixed the error handling. + +### 2015-06-01 + +By computing the bit length of the LZMA operations I was able to +improve the greedy algorithm implementation. By using an 8 MByte buffer +the compression rate was not as good as for xz but already better then +gzip default. + +Compression is currently slow, but this is something we will be able to +improve over time. + +### 2015-05-26 + +Checked the license of ogier/pflag. The binary lzmago binary should +include the license terms for the pflag library. + +I added the endorsement clause as used by Google for the Go sources the +LICENSE file. + +### 2015-05-22 + +The package lzb contains now the basic implementation for creating or +reading LZMA byte streams. It allows the support for the implementation +of the DAG-shortest-path algorithm for the compression function. + +### 2015-04-23 + +Completed yesterday the lzbase classes. I'm a little bit concerned that +using the components may require too much code, but on the other hand +there is a lot of flexibility. + +### 2015-04-22 + +Implemented Reader and Writer during the Bayern game against Porto. The +second half gave me enough time. + +### 2015-04-21 + +While showering today morning I discovered that the design for OpEncoder +and OpDecoder doesn't work, because encoding/decoding might depend on +the current status of the dictionary. This is not exactly the right way +to start the day. + +Therefore we need to keep the Reader and Writer design. This time around +we simplify it by ignoring size limits. These can be added by wrappers +around the Reader and Writer interfaces. The Parameters type isn't +needed anymore. + +However I will implement a ReaderState and WriterState type to use +static typing to ensure the right State object is combined with the +right lzbase.Reader and lzbase.Writer. + +As a start I have implemented ReaderState and WriterState to ensure +that the state for reading is only used by readers and WriterState only +used by Writers. + +### 2015-04-20 + +Today I implemented the OpDecoder and tested OpEncoder and OpDecoder. + +### 2015-04-08 + +Came up with a new simplified design for lzbase. I implemented already +the type State that replaces OpCodec. + +### 2015-04-06 + +The new lzma package is now fully usable and lzmago is using it now. The +old lzma package has been completely removed. + +### 2015-04-05 + +Implemented lzma.Reader and tested it. + +### 2015-04-04 + +Implemented baseReader by adapting code form lzma.Reader. + +### 2015-04-03 + +The opCodec has been copied yesterday to lzma2. opCodec has a high +number of dependencies on other files in lzma2. Therefore I had to copy +almost all files from lzma. + +### 2015-03-31 + +Removed only a TODO item. + +However in Francesco Campoy's presentation "Go for Javaneros +(Javaïstes?)" is the the idea that using an embedded field E, all the +methods of E will be defined on T. If E is an interface T satisfies E. + +https://talks.golang.org/2014/go4java.slide#51 + +I have never used this, but it seems to be a cool idea. + +### 2015-03-30 + +Finished the type writerDict and wrote a simple test. + +### 2015-03-25 + +I started to implement the writerDict. + +### 2015-03-24 + +After thinking long about the LZMA2 code and several false starts, I +have now a plan to create a self-sufficient lzma2 package that supports +the classic LZMA format as well as LZMA2. The core idea is to support a +baseReader and baseWriter type that support the basic LZMA stream +without any headers. Both types must support the reuse of dictionaries +and the opCodec. + +### 2015-01-10 + +1. Implemented simple lzmago tool +2. Tested tool against large 4.4G file + - compression worked correctly; tested decompression with lzma + - decompression hits a full buffer condition +3. Fixed a bug in the compressor and wrote a test for it +4. Executed full cycle for 4.4 GB file; performance can be improved ;-) + +### 2015-01-11 + +- Release v0.2 because of the working LZMA encoder and decoder diff --git a/vendor/github.com/ulikunitz/xz/bits.go b/vendor/github.com/ulikunitz/xz/bits.go new file mode 100644 index 0000000..fadc1a5 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/bits.go @@ -0,0 +1,74 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "errors" + "io" +) + +// putUint32LE puts the little-endian representation of x into the first +// four bytes of p. +func putUint32LE(p []byte, x uint32) { + p[0] = byte(x) + p[1] = byte(x >> 8) + p[2] = byte(x >> 16) + p[3] = byte(x >> 24) +} + +// putUint64LE puts the little-endian representation of x into the first +// eight bytes of p. +func putUint64LE(p []byte, x uint64) { + p[0] = byte(x) + p[1] = byte(x >> 8) + p[2] = byte(x >> 16) + p[3] = byte(x >> 24) + p[4] = byte(x >> 32) + p[5] = byte(x >> 40) + p[6] = byte(x >> 48) + p[7] = byte(x >> 56) +} + +// uint32LE converts a little endian representation to an uint32 value. +func uint32LE(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | + uint32(p[3])<<24 +} + +// putUvarint puts a uvarint representation of x into the byte slice. +func putUvarint(p []byte, x uint64) int { + i := 0 + for x >= 0x80 { + p[i] = byte(x) | 0x80 + x >>= 7 + i++ + } + p[i] = byte(x) + return i + 1 +} + +// errOverflow indicates an overflow of the 64-bit unsigned integer. +var errOverflowU64 = errors.New("xz: uvarint overflows 64-bit unsigned integer") + +// readUvarint reads a uvarint from the given byte reader. +func readUvarint(r io.ByteReader) (x uint64, n int, err error) { + var s uint + i := 0 + for { + b, err := r.ReadByte() + if err != nil { + return x, i, err + } + i++ + if b < 0x80 { + if i > 10 || i == 10 && b > 1 { + return x, i, errOverflowU64 + } + return x | uint64(b)<.... + +The xb copyright command adds a copyright remark to all go files below path. + + -h prints this message and exits +` + +func crUsage(w io.Writer) { + fmt.Fprint(w, crUsageString) +} + +const copyrightText = ` +Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file. +` + +func goComment(text string) string { + buf := new(bytes.Buffer) + scanner := bufio.NewScanner(strings.NewReader(text)) + var err error + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if len(s) == 0 { + continue + } + if _, err = fmt.Fprintln(buf, "//", s); err != nil { + panic(err) + } + } + if err = scanner.Err(); err != nil { + panic(err) + } + if _, err = fmt.Fprintln(buf); err != nil { + panic(err) + } + return buf.String() +} + +var goCopyright = goComment(copyrightText) + +func addCopyright(path string) (err error) { + log.Printf("adding copyright to %s", path) + src, err := os.Open(path) + if err != nil { + return err + } + defer func() { + cerr := src.Close() + if cerr != nil && err == nil { + err = cerr + } + }() + newPath := path + ".new" + dst, err := os.Create(newPath) + if err != nil { + return err + } + defer func() { + cerr := dst.Close() + if cerr != nil && err == nil { + err = cerr + } + }() + out := bufio.NewWriter(dst) + fmt.Fprint(out, goCopyright) + scanner := bufio.NewScanner(src) + line := 0 + del := false + for scanner.Scan() { + line++ + txt := scanner.Text() + if line == 1 && strings.Contains(txt, "Copyright") { + del = true + continue + } + if del { + s := strings.TrimSpace(txt) + if len(s) == 0 { + del = false + } + continue + } + fmt.Fprintln(out, txt) + } + if err = scanner.Err(); err != nil { + return err + } + if err = out.Flush(); err != nil { + return + } + err = os.Rename(newPath, path) + return +} + +func walkCopyrights(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if !strings.HasSuffix(info.Name(), ".go") { + return nil + } + return addCopyright(path) +} + +func copyright() { + cmdName := os.Args[0] + log.SetPrefix(fmt.Sprintf("%s: ", cmdName)) + log.SetFlags(0) + + flag.CommandLine = flag.NewFlagSet(cmdName, flag.ExitOnError) + flag.Usage = func() { crUsage(os.Stderr); os.Exit(1) } + + help := flag.Bool("h", false, "") + + flag.Parse() + + if *help { + crUsage(os.Stdout) + os.Exit(0) + } + + for _, path := range flag.Args() { + fi, err := os.Stat(path) + if err != nil { + log.Print(err) + continue + } + if !fi.IsDir() { + log.Printf("%s is not a directory", path) + continue + } + if err = filepath.Walk(path, walkCopyrights); err != nil { + log.Fatalf("%s error %s", path, err) + } + } +} diff --git a/vendor/github.com/ulikunitz/xz/crc.go b/vendor/github.com/ulikunitz/xz/crc.go new file mode 100644 index 0000000..b44dca9 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/crc.go @@ -0,0 +1,54 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "hash" + "hash/crc32" + "hash/crc64" +) + +// crc32Hash implements the hash.Hash32 interface with Sum returning the +// crc32 value in little-endian encoding. +type crc32Hash struct { + hash.Hash32 +} + +// Sum returns the crc32 value as little endian. +func (h crc32Hash) Sum(b []byte) []byte { + p := make([]byte, 4) + putUint32LE(p, h.Hash32.Sum32()) + b = append(b, p...) + return b +} + +// newCRC32 returns a CRC-32 hash that returns the 64-bit value in +// little-endian encoding using the IEEE polynomial. +func newCRC32() hash.Hash { + return crc32Hash{Hash32: crc32.NewIEEE()} +} + +// crc64Hash implements the Hash64 interface with Sum returning the +// CRC-64 value in little-endian encoding. +type crc64Hash struct { + hash.Hash64 +} + +// Sum returns the CRC-64 value in little-endian encoding. +func (h crc64Hash) Sum(b []byte) []byte { + p := make([]byte, 8) + putUint64LE(p, h.Hash64.Sum64()) + b = append(b, p...) + return b +} + +// crc64Table is used to create a CRC-64 hash. +var crc64Table = crc64.MakeTable(crc64.ECMA) + +// newCRC64 returns a CRC-64 hash that returns the 64-bit value in +// little-endian encoding using the ECMA polynomial. +func newCRC64() hash.Hash { + return crc64Hash{Hash64: crc64.New(crc64Table)} +} diff --git a/vendor/github.com/ulikunitz/xz/example.go b/vendor/github.com/ulikunitz/xz/example.go new file mode 100644 index 0000000..855e60a --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/example.go @@ -0,0 +1,40 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "io" + "log" + "os" + + "github.com/ulikunitz/xz" +) + +func main() { + const text = "The quick brown fox jumps over the lazy dog.\n" + var buf bytes.Buffer + // compress text + w, err := xz.NewWriter(&buf) + if err != nil { + log.Fatalf("xz.NewWriter error %s", err) + } + if _, err := io.WriteString(w, text); err != nil { + log.Fatalf("WriteString error %s", err) + } + if err := w.Close(); err != nil { + log.Fatalf("w.Close error %s", err) + } + // decompress buffer and write output to stdout + r, err := xz.NewReader(&buf) + if err != nil { + log.Fatalf("NewReader error %s", err) + } + if _, err = io.Copy(os.Stdout, r); err != nil { + log.Fatalf("io.Copy error %s", err) + } +} diff --git a/vendor/github.com/ulikunitz/xz/format.go b/vendor/github.com/ulikunitz/xz/format.go new file mode 100644 index 0000000..798159c --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/format.go @@ -0,0 +1,728 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// allZeros checks whether a given byte slice has only zeros. +func allZeros(p []byte) bool { + for _, c := range p { + if c != 0 { + return false + } + } + return true +} + +// padLen returns the length of the padding required for the given +// argument. +func padLen(n int64) int { + k := int(n % 4) + if k > 0 { + k = 4 - k + } + return k +} + +/*** Header ***/ + +// headerMagic stores the magic bytes for the header +var headerMagic = []byte{0xfd, '7', 'z', 'X', 'Z', 0x00} + +// HeaderLen provides the length of the xz file header. +const HeaderLen = 12 + +// Constants for the checksum methods supported by xz. +const ( + CRC32 byte = 0x1 + CRC64 = 0x4 + SHA256 = 0xa +) + +// errInvalidFlags indicates that flags are invalid. +var errInvalidFlags = errors.New("xz: invalid flags") + +// verifyFlags returns the error errInvalidFlags if the value is +// invalid. +func verifyFlags(flags byte) error { + switch flags { + case CRC32, CRC64, SHA256: + return nil + default: + return errInvalidFlags + } +} + +// flagstrings maps flag values to strings. +var flagstrings = map[byte]string{ + CRC32: "CRC-32", + CRC64: "CRC-64", + SHA256: "SHA-256", +} + +// flagString returns the string representation for the given flags. +func flagString(flags byte) string { + s, ok := flagstrings[flags] + if !ok { + return "invalid" + } + return s +} + +// newHashFunc returns a function that creates hash instances for the +// hash method encoded in flags. +func newHashFunc(flags byte) (newHash func() hash.Hash, err error) { + switch flags { + case CRC32: + newHash = newCRC32 + case CRC64: + newHash = newCRC64 + case SHA256: + newHash = sha256.New + default: + err = errInvalidFlags + } + return +} + +// header provides the actual content of the xz file header: the flags. +type header struct { + flags byte +} + +// Errors returned by readHeader. +var errHeaderMagic = errors.New("xz: invalid header magic bytes") + +// ValidHeader checks whether data is a correct xz file header. The +// length of data must be HeaderLen. +func ValidHeader(data []byte) bool { + var h header + err := h.UnmarshalBinary(data) + return err == nil +} + +// String returns a string representation of the flags. +func (h header) String() string { + return flagString(h.flags) +} + +// UnmarshalBinary reads header from the provided data slice. +func (h *header) UnmarshalBinary(data []byte) error { + // header length + if len(data) != HeaderLen { + return errors.New("xz: wrong file header length") + } + + // magic header + if !bytes.Equal(headerMagic, data[:6]) { + return errHeaderMagic + } + + // checksum + crc := crc32.NewIEEE() + crc.Write(data[6:8]) + if uint32LE(data[8:]) != crc.Sum32() { + return errors.New("xz: invalid checksum for file header") + } + + // stream flags + if data[6] != 0 { + return errInvalidFlags + } + flags := data[7] + if err := verifyFlags(flags); err != nil { + return err + } + + h.flags = flags + return nil +} + +// MarshalBinary generates the xz file header. +func (h *header) MarshalBinary() (data []byte, err error) { + if err = verifyFlags(h.flags); err != nil { + return nil, err + } + + data = make([]byte, 12) + copy(data, headerMagic) + data[7] = h.flags + + crc := crc32.NewIEEE() + crc.Write(data[6:8]) + putUint32LE(data[8:], crc.Sum32()) + + return data, nil +} + +/*** Footer ***/ + +// footerLen defines the length of the footer. +const footerLen = 12 + +// footerMagic contains the footer magic bytes. +var footerMagic = []byte{'Y', 'Z'} + +// footer represents the content of the xz file footer. +type footer struct { + indexSize int64 + flags byte +} + +// String prints a string representation of the footer structure. +func (f footer) String() string { + return fmt.Sprintf("%s index size %d", flagString(f.flags), f.indexSize) +} + +// Minimum and maximum for the size of the index (backward size). +const ( + minIndexSize = 4 + maxIndexSize = (1 << 32) * 4 +) + +// MarshalBinary converts footer values into an xz file footer. Note +// that the footer value is checked for correctness. +func (f *footer) MarshalBinary() (data []byte, err error) { + if err = verifyFlags(f.flags); err != nil { + return nil, err + } + if !(minIndexSize <= f.indexSize && f.indexSize <= maxIndexSize) { + return nil, errors.New("xz: index size out of range") + } + if f.indexSize%4 != 0 { + return nil, errors.New( + "xz: index size not aligned to four bytes") + } + + data = make([]byte, footerLen) + + // backward size (index size) + s := (f.indexSize / 4) - 1 + putUint32LE(data[4:], uint32(s)) + // flags + data[9] = f.flags + // footer magic + copy(data[10:], footerMagic) + + // CRC-32 + crc := crc32.NewIEEE() + crc.Write(data[4:10]) + putUint32LE(data, crc.Sum32()) + + return data, nil +} + +// UnmarshalBinary sets the footer value by unmarshalling an xz file +// footer. +func (f *footer) UnmarshalBinary(data []byte) error { + if len(data) != footerLen { + return errors.New("xz: wrong footer length") + } + + // magic bytes + if !bytes.Equal(data[10:], footerMagic) { + return errors.New("xz: footer magic invalid") + } + + // CRC-32 + crc := crc32.NewIEEE() + crc.Write(data[4:10]) + if uint32LE(data) != crc.Sum32() { + return errors.New("xz: footer checksum error") + } + + var g footer + // backward size (index size) + g.indexSize = (int64(uint32LE(data[4:])) + 1) * 4 + + // flags + if data[8] != 0 { + return errInvalidFlags + } + g.flags = data[9] + if err := verifyFlags(g.flags); err != nil { + return err + } + + *f = g + return nil +} + +/*** Block Header ***/ + +// blockHeader represents the content of an xz block header. +type blockHeader struct { + compressedSize int64 + uncompressedSize int64 + filters []filter +} + +// String converts the block header into a string. +func (h blockHeader) String() string { + var buf bytes.Buffer + first := true + if h.compressedSize >= 0 { + fmt.Fprintf(&buf, "compressed size %d", h.compressedSize) + first = false + } + if h.uncompressedSize >= 0 { + if !first { + buf.WriteString(" ") + } + fmt.Fprintf(&buf, "uncompressed size %d", h.uncompressedSize) + first = false + } + for _, f := range h.filters { + if !first { + buf.WriteString(" ") + } + fmt.Fprintf(&buf, "filter %s", f) + first = false + } + return buf.String() +} + +// Masks for the block flags. +const ( + filterCountMask = 0x03 + compressedSizePresent = 0x40 + uncompressedSizePresent = 0x80 + reservedBlockFlags = 0x3C +) + +// errIndexIndicator signals that an index indicator (0x00) has been found +// instead of an expected block header indicator. +var errIndexIndicator = errors.New("xz: found index indicator") + +// readBlockHeader reads the block header. +func readBlockHeader(r io.Reader) (h *blockHeader, n int, err error) { + var buf bytes.Buffer + buf.Grow(20) + + // block header size + z, err := io.CopyN(&buf, r, 1) + n = int(z) + if err != nil { + return nil, n, err + } + s := buf.Bytes()[0] + if s == 0 { + return nil, n, errIndexIndicator + } + + // read complete header + headerLen := (int(s) + 1) * 4 + buf.Grow(headerLen - 1) + z, err = io.CopyN(&buf, r, int64(headerLen-1)) + n += int(z) + if err != nil { + return nil, n, err + } + + // unmarshal block header + h = new(blockHeader) + if err = h.UnmarshalBinary(buf.Bytes()); err != nil { + return nil, n, err + } + + return h, n, nil +} + +// readSizeInBlockHeader reads the uncompressed or compressed size +// fields in the block header. The present value informs the function +// whether the respective field is actually present in the header. +func readSizeInBlockHeader(r io.ByteReader, present bool) (n int64, err error) { + if !present { + return -1, nil + } + x, _, err := readUvarint(r) + if err != nil { + return 0, err + } + if x >= 1<<63 { + return 0, errors.New("xz: size overflow in block header") + } + return int64(x), nil +} + +// UnmarshalBinary unmarshals the block header. +func (h *blockHeader) UnmarshalBinary(data []byte) error { + // Check header length + s := data[0] + if data[0] == 0 { + return errIndexIndicator + } + headerLen := (int(s) + 1) * 4 + if len(data) != headerLen { + return fmt.Errorf("xz: data length %d; want %d", len(data), + headerLen) + } + n := headerLen - 4 + + // Check CRC-32 + crc := crc32.NewIEEE() + crc.Write(data[:n]) + if crc.Sum32() != uint32LE(data[n:]) { + return errors.New("xz: checksum error for block header") + } + + // Block header flags + flags := data[1] + if flags&reservedBlockFlags != 0 { + return errors.New("xz: reserved block header flags set") + } + + r := bytes.NewReader(data[2:n]) + + // Compressed size + var err error + h.compressedSize, err = readSizeInBlockHeader( + r, flags&compressedSizePresent != 0) + if err != nil { + return err + } + + // Uncompressed size + h.uncompressedSize, err = readSizeInBlockHeader( + r, flags&uncompressedSizePresent != 0) + if err != nil { + return err + } + + h.filters, err = readFilters(r, int(flags&filterCountMask)+1) + if err != nil { + return err + } + + // Check padding + // Since headerLen is a multiple of 4 we don't need to check + // alignment. + k := r.Len() + // The standard spec says that the padding should have not more + // than 3 bytes. However we found paddings of 4 or 5 in the + // wild. See https://github.com/ulikunitz/xz/pull/11 and + // https://github.com/ulikunitz/xz/issues/15 + // + // The only reasonable approach seems to be to ignore the + // padding size. We still check that all padding bytes are zero. + if !allZeros(data[n-k : n]) { + return errPadding + } + return nil +} + +// MarshalBinary marshals the binary header. +func (h *blockHeader) MarshalBinary() (data []byte, err error) { + if !(minFilters <= len(h.filters) && len(h.filters) <= maxFilters) { + return nil, errors.New("xz: filter count wrong") + } + for i, f := range h.filters { + if i < len(h.filters)-1 { + if f.id() == lzmaFilterID { + return nil, errors.New( + "xz: LZMA2 filter is not the last") + } + } else { + // last filter + if f.id() != lzmaFilterID { + return nil, errors.New("xz: " + + "last filter must be the LZMA2 filter") + } + } + } + + var buf bytes.Buffer + // header size must set at the end + buf.WriteByte(0) + + // flags + flags := byte(len(h.filters) - 1) + if h.compressedSize >= 0 { + flags |= compressedSizePresent + } + if h.uncompressedSize >= 0 { + flags |= uncompressedSizePresent + } + buf.WriteByte(flags) + + p := make([]byte, 10) + if h.compressedSize >= 0 { + k := putUvarint(p, uint64(h.compressedSize)) + buf.Write(p[:k]) + } + if h.uncompressedSize >= 0 { + k := putUvarint(p, uint64(h.uncompressedSize)) + buf.Write(p[:k]) + } + + for _, f := range h.filters { + fp, err := f.MarshalBinary() + if err != nil { + return nil, err + } + buf.Write(fp) + } + + // padding + for i := padLen(int64(buf.Len())); i > 0; i-- { + buf.WriteByte(0) + } + + // crc place holder + buf.Write(p[:4]) + + data = buf.Bytes() + if len(data)%4 != 0 { + panic("data length not aligned") + } + s := len(data)/4 - 1 + if !(1 < s && s <= 255) { + panic("wrong block header size") + } + data[0] = byte(s) + + crc := crc32.NewIEEE() + crc.Write(data[:len(data)-4]) + putUint32LE(data[len(data)-4:], crc.Sum32()) + + return data, nil +} + +// Constants used for marshalling and unmarshalling filters in the xz +// block header. +const ( + minFilters = 1 + maxFilters = 4 + minReservedID = 1 << 62 +) + +// filter represents a filter in the block header. +type filter interface { + id() uint64 + UnmarshalBinary(data []byte) error + MarshalBinary() (data []byte, err error) + reader(r io.Reader, c *ReaderConfig) (fr io.Reader, err error) + writeCloser(w io.WriteCloser, c *WriterConfig) (fw io.WriteCloser, err error) + // filter must be last filter + last() bool +} + +// readFilter reads a block filter from the block header. At this point +// in time only the LZMA2 filter is supported. +func readFilter(r io.Reader) (f filter, err error) { + br := lzma.ByteReader(r) + + // index + id, _, err := readUvarint(br) + if err != nil { + return nil, err + } + + var data []byte + switch id { + case lzmaFilterID: + data = make([]byte, lzmaFilterLen) + data[0] = lzmaFilterID + if _, err = io.ReadFull(r, data[1:]); err != nil { + return nil, err + } + f = new(lzmaFilter) + default: + if id >= minReservedID { + return nil, errors.New( + "xz: reserved filter id in block stream header") + } + return nil, errors.New("xz: invalid filter id") + } + if err = f.UnmarshalBinary(data); err != nil { + return nil, err + } + return f, err +} + +// readFilters reads count filters. At this point in time only the count +// 1 is supported. +func readFilters(r io.Reader, count int) (filters []filter, err error) { + if count != 1 { + return nil, errors.New("xz: unsupported filter count") + } + f, err := readFilter(r) + if err != nil { + return nil, err + } + return []filter{f}, err +} + +// writeFilters writes the filters. +func writeFilters(w io.Writer, filters []filter) (n int, err error) { + for _, f := range filters { + p, err := f.MarshalBinary() + if err != nil { + return n, err + } + k, err := w.Write(p) + n += k + if err != nil { + return n, err + } + } + return n, nil +} + +/*** Index ***/ + +// record describes a block in the xz file index. +type record struct { + unpaddedSize int64 + uncompressedSize int64 +} + +// readRecord reads an index record. +func readRecord(r io.ByteReader) (rec record, n int, err error) { + u, k, err := readUvarint(r) + n += k + if err != nil { + return rec, n, err + } + rec.unpaddedSize = int64(u) + if rec.unpaddedSize < 0 { + return rec, n, errors.New("xz: unpadded size negative") + } + + u, k, err = readUvarint(r) + n += k + if err != nil { + return rec, n, err + } + rec.uncompressedSize = int64(u) + if rec.uncompressedSize < 0 { + return rec, n, errors.New("xz: uncompressed size negative") + } + + return rec, n, nil +} + +// MarshalBinary converts an index record in its binary encoding. +func (rec *record) MarshalBinary() (data []byte, err error) { + // maximum length of a uvarint is 10 + p := make([]byte, 20) + n := putUvarint(p, uint64(rec.unpaddedSize)) + n += putUvarint(p[n:], uint64(rec.uncompressedSize)) + return p[:n], nil +} + +// writeIndex writes the index, a sequence of records. +func writeIndex(w io.Writer, index []record) (n int64, err error) { + crc := crc32.NewIEEE() + mw := io.MultiWriter(w, crc) + + // index indicator + k, err := mw.Write([]byte{0}) + n += int64(k) + if err != nil { + return n, err + } + + // number of records + p := make([]byte, 10) + k = putUvarint(p, uint64(len(index))) + k, err = mw.Write(p[:k]) + n += int64(k) + if err != nil { + return n, err + } + + // list of records + for _, rec := range index { + p, err := rec.MarshalBinary() + if err != nil { + return n, err + } + k, err = mw.Write(p) + n += int64(k) + if err != nil { + return n, err + } + } + + // index padding + k, err = mw.Write(make([]byte, padLen(int64(n)))) + n += int64(k) + if err != nil { + return n, err + } + + // crc32 checksum + putUint32LE(p, crc.Sum32()) + k, err = w.Write(p[:4]) + n += int64(k) + + return n, err +} + +// readIndexBody reads the index from the reader. It assumes that the +// index indicator has already been read. +func readIndexBody(r io.Reader) (records []record, n int64, err error) { + crc := crc32.NewIEEE() + // index indicator + crc.Write([]byte{0}) + + br := lzma.ByteReader(io.TeeReader(r, crc)) + + // number of records + u, k, err := readUvarint(br) + n += int64(k) + if err != nil { + return nil, n, err + } + recLen := int(u) + if recLen < 0 || uint64(recLen) != u { + return nil, n, errors.New("xz: record number overflow") + } + + // list of records + records = make([]record, recLen) + for i := range records { + records[i], k, err = readRecord(br) + n += int64(k) + if err != nil { + return nil, n, err + } + } + + p := make([]byte, padLen(int64(n+1)), 4) + k, err = io.ReadFull(br.(io.Reader), p) + n += int64(k) + if err != nil { + return nil, n, err + } + if !allZeros(p) { + return nil, n, errors.New("xz: non-zero byte in index padding") + } + + // crc32 + s := crc.Sum32() + p = p[:4] + k, err = io.ReadFull(br.(io.Reader), p) + n += int64(k) + if err != nil { + return records, n, err + } + if uint32LE(p) != s { + return nil, n, errors.New("xz: wrong checksum for index") + } + + return records, n, nil +} diff --git a/vendor/github.com/ulikunitz/xz/fox.xz b/vendor/github.com/ulikunitz/xz/fox.xz new file mode 100644 index 0000000..4b820bd Binary files /dev/null and b/vendor/github.com/ulikunitz/xz/fox.xz differ diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go b/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go new file mode 100644 index 0000000..a328878 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go @@ -0,0 +1,181 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hash + +// CyclicPoly provides a cyclic polynomial rolling hash. +type CyclicPoly struct { + h uint64 + p []uint64 + i int +} + +// ror rotates the unsigned 64-bit integer to right. The argument s must be +// less than 64. +func ror(x uint64, s uint) uint64 { + return (x >> s) | (x << (64 - s)) +} + +// NewCyclicPoly creates a new instance of the CyclicPoly structure. The +// argument n gives the number of bytes for which a hash will be executed. +// This number must be positive; the method panics if this isn't the case. +func NewCyclicPoly(n int) *CyclicPoly { + if n < 1 { + panic("argument n must be positive") + } + return &CyclicPoly{p: make([]uint64, 0, n)} +} + +// Len returns the length of the byte sequence for which a hash is generated. +func (r *CyclicPoly) Len() int { + return cap(r.p) +} + +// RollByte hashes the next byte and returns a hash value. The complete becomes +// available after at least Len() bytes have been hashed. +func (r *CyclicPoly) RollByte(x byte) uint64 { + y := hash[x] + if len(r.p) < cap(r.p) { + r.h = ror(r.h, 1) ^ y + r.p = append(r.p, y) + } else { + r.h ^= ror(r.p[r.i], uint(cap(r.p)-1)) + r.h = ror(r.h, 1) ^ y + r.p[r.i] = y + r.i = (r.i + 1) % cap(r.p) + } + return r.h +} + +// Stores the hash for the individual bytes. +var hash = [256]uint64{ + 0x2e4fc3f904065142, 0xc790984cfbc99527, + 0x879f95eb8c62f187, 0x3b61be86b5021ef2, + 0x65a896a04196f0a5, 0xc5b307b80470b59e, + 0xd3bff376a70df14b, 0xc332f04f0b3f1701, + 0x753b5f0e9abf3e0d, 0xb41538fdfe66ef53, + 0x1906a10c2c1c0208, 0xfb0c712a03421c0d, + 0x38be311a65c9552b, 0xfee7ee4ca6445c7e, + 0x71aadeded184f21e, 0xd73426fccda23b2d, + 0x29773fb5fb9600b5, 0xce410261cd32981a, + 0xfe2848b3c62dbc2d, 0x459eaaff6e43e11c, + 0xc13e35fc9c73a887, 0xf30ed5c201e76dbc, + 0xa5f10b3910482cea, 0x2945d59be02dfaad, + 0x06ee334ff70571b5, 0xbabf9d8070f44380, + 0xee3e2e9912ffd27c, 0x2a7118d1ea6b8ea7, + 0x26183cb9f7b1664c, 0xea71dac7da068f21, + 0xea92eca5bd1d0bb7, 0x415595862defcd75, + 0x248a386023c60648, 0x9cf021ab284b3c8a, + 0xfc9372df02870f6c, 0x2b92d693eeb3b3fc, + 0x73e799d139dc6975, 0x7b15ae312486363c, + 0xb70e5454a2239c80, 0x208e3fb31d3b2263, + 0x01f563cabb930f44, 0x2ac4533d2a3240d8, + 0x84231ed1064f6f7c, 0xa9f020977c2a6d19, + 0x213c227271c20122, 0x09fe8a9a0a03d07a, + 0x4236dc75bcaf910c, 0x460a8b2bead8f17e, + 0xd9b27be1aa07055f, 0xd202d5dc4b11c33e, + 0x70adb010543bea12, 0xcdae938f7ea6f579, + 0x3f3d870208672f4d, 0x8e6ccbce9d349536, + 0xe4c0871a389095ae, 0xf5f2a49152bca080, + 0x9a43f9b97269934e, 0xc17b3753cb6f475c, + 0xd56d941e8e206bd4, 0xac0a4f3e525eda00, + 0xa06d5a011912a550, 0x5537ed19537ad1df, + 0xa32fe713d611449d, 0x2a1d05b47c3b579f, + 0x991d02dbd30a2a52, 0x39e91e7e28f93eb0, + 0x40d06adb3e92c9ac, 0x9b9d3afde1c77c97, + 0x9a3f3f41c02c616f, 0x22ecd4ba00f60c44, + 0x0b63d5d801708420, 0x8f227ca8f37ffaec, + 0x0256278670887c24, 0x107e14877dbf540b, + 0x32c19f2786ac1c05, 0x1df5b12bb4bc9c61, + 0xc0cac129d0d4c4e2, 0x9fdb52ee9800b001, + 0x31f601d5d31c48c4, 0x72ff3c0928bcaec7, + 0xd99264421147eb03, 0x535a2d6d38aefcfe, + 0x6ba8b4454a916237, 0xfa39366eaae4719c, + 0x10f00fd7bbb24b6f, 0x5bd23185c76c84d4, + 0xb22c3d7e1b00d33f, 0x3efc20aa6bc830a8, + 0xd61c2503fe639144, 0x30ce625441eb92d3, + 0xe5d34cf359e93100, 0xa8e5aa13f2b9f7a5, + 0x5c2b8d851ca254a6, 0x68fb6c5e8b0d5fdf, + 0xc7ea4872c96b83ae, 0x6dd5d376f4392382, + 0x1be88681aaa9792f, 0xfef465ee1b6c10d9, + 0x1f98b65ed43fcb2e, 0x4d1ca11eb6e9a9c9, + 0x7808e902b3857d0b, 0x171c9c4ea4607972, + 0x58d66274850146df, 0x42b311c10d3981d1, + 0x647fa8c621c41a4c, 0xf472771c66ddfedc, + 0x338d27e3f847b46b, 0x6402ce3da97545ce, + 0x5162db616fc38638, 0x9c83be97bc22a50e, + 0x2d3d7478a78d5e72, 0xe621a9b938fd5397, + 0x9454614eb0f81c45, 0x395fb6e742ed39b6, + 0x77dd9179d06037bf, 0xc478d0fee4d2656d, + 0x35d9d6cb772007af, 0x83a56e92c883f0f6, + 0x27937453250c00a1, 0x27bd6ebc3a46a97d, + 0x9f543bf784342d51, 0xd158f38c48b0ed52, + 0x8dd8537c045f66b4, 0x846a57230226f6d5, + 0x6b13939e0c4e7cdf, 0xfca25425d8176758, + 0x92e5fc6cd52788e6, 0x9992e13d7a739170, + 0x518246f7a199e8ea, 0xf104c2a71b9979c7, + 0x86b3ffaabea4768f, 0x6388061cf3e351ad, + 0x09d9b5295de5bbb5, 0x38bf1638c2599e92, + 0x1d759846499e148d, 0x4c0ff015e5f96ef4, + 0xa41a94cfa270f565, 0x42d76f9cb2326c0b, + 0x0cf385dd3c9c23ba, 0x0508a6c7508d6e7a, + 0x337523aabbe6cf8d, 0x646bb14001d42b12, + 0xc178729d138adc74, 0xf900ef4491f24086, + 0xee1a90d334bb5ac4, 0x9755c92247301a50, + 0xb999bf7c4ff1b610, 0x6aeeb2f3b21e8fc9, + 0x0fa8084cf91ac6ff, 0x10d226cf136e6189, + 0xd302057a07d4fb21, 0x5f03800e20a0fcc3, + 0x80118d4ae46bd210, 0x58ab61a522843733, + 0x51edd575c5432a4b, 0x94ee6ff67f9197f7, + 0x765669e0e5e8157b, 0xa5347830737132f0, + 0x3ba485a69f01510c, 0x0b247d7b957a01c3, + 0x1b3d63449fd807dc, 0x0fdc4721c30ad743, + 0x8b535ed3829b2b14, 0xee41d0cad65d232c, + 0xe6a99ed97a6a982f, 0x65ac6194c202003d, + 0x692accf3a70573eb, 0xcc3c02c3e200d5af, + 0x0d419e8b325914a3, 0x320f160f42c25e40, + 0x00710d647a51fe7a, 0x3c947692330aed60, + 0x9288aa280d355a7a, 0xa1806a9b791d1696, + 0x5d60e38496763da1, 0x6c69e22e613fd0f4, + 0x977fc2a5aadffb17, 0xfb7bd063fc5a94ba, + 0x460c17992cbaece1, 0xf7822c5444d3297f, + 0x344a9790c69b74aa, 0xb80a42e6cae09dce, + 0x1b1361eaf2b1e757, 0xd84c1e758e236f01, + 0x88e0b7be347627cc, 0x45246009b7a99490, + 0x8011c6dd3fe50472, 0xc341d682bffb99d7, + 0x2511be93808e2d15, 0xd5bc13d7fd739840, + 0x2a3cd030679ae1ec, 0x8ad9898a4b9ee157, + 0x3245fef0a8eaf521, 0x3d6d8dbbb427d2b0, + 0x1ed146d8968b3981, 0x0c6a28bf7d45f3fc, + 0x4a1fd3dbcee3c561, 0x4210ff6a476bf67e, + 0xa559cce0d9199aac, 0xde39d47ef3723380, + 0xe5b69d848ce42e35, 0xefa24296f8e79f52, + 0x70190b59db9a5afc, 0x26f166cdb211e7bf, + 0x4deaf2df3c6b8ef5, 0xf171dbdd670f1017, + 0xb9059b05e9420d90, 0x2f0da855c9388754, + 0x611d5e9ab77949cc, 0x2912038ac01163f4, + 0x0231df50402b2fba, 0x45660fc4f3245f58, + 0xb91cc97c7c8dac50, 0xb72d2aafe4953427, + 0xfa6463f87e813d6b, 0x4515f7ee95d5c6a2, + 0x1310e1c1a48d21c3, 0xad48a7810cdd8544, + 0x4d5bdfefd5c9e631, 0xa43ed43f1fdcb7de, + 0xe70cfc8fe1ee9626, 0xef4711b0d8dda442, + 0xb80dd9bd4dab6c93, 0xa23be08d31ba4d93, + 0x9b37db9d0335a39c, 0x494b6f870f5cfebc, + 0x6d1b3c1149dda943, 0x372c943a518c1093, + 0xad27af45e77c09c4, 0x3b6f92b646044604, + 0xac2917909f5fcf4f, 0x2069a60e977e5557, + 0x353a469e71014de5, 0x24be356281f55c15, + 0x2b6d710ba8e9adea, 0x404ad1751c749c29, + 0xed7311bf23d7f185, 0xba4f6976b4acc43e, + 0x32d7198d2bc39000, 0xee667019014d6e01, + 0x494ef3e128d14c83, 0x1f95a152baecd6be, + 0x201648dff1f483a5, 0x68c28550c8384af6, + 0x5fc834a6824a7f48, 0x7cd06cb7365eaf28, + 0xd82bbd95e9b30909, 0x234f0d1694c53f6d, + 0xd2fb7f4a96d83f4a, 0xff0d5da83acac05e, + 0xf8f6b97f5585080a, 0x74236084be57b95b, + 0xa25e40c03bbc36ad, 0x6b6e5c14ce88465b, + 0x4378ffe93e1528c5, 0x94ca92a17118e2d2, +} diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/doc.go b/vendor/github.com/ulikunitz/xz/internal/hash/doc.go new file mode 100644 index 0000000..f99ec22 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/doc.go @@ -0,0 +1,14 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package hash provides rolling hashes. + +Rolling hashes have to be used for maintaining the positions of n-byte +sequences in the dictionary buffer. + +The package provides currently the Rabin-Karp rolling hash and a Cyclic +Polynomial hash. Both support the Hashes method to be used with an interface. +*/ +package hash diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go b/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go new file mode 100644 index 0000000..58635b1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go @@ -0,0 +1,66 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hash + +// A is the default constant for Robin-Karp rolling hash. This is a random +// prime. +const A = 0x97b548add41d5da1 + +// RabinKarp supports the computation of a rolling hash. +type RabinKarp struct { + A uint64 + // a^n + aOldest uint64 + h uint64 + p []byte + i int +} + +// NewRabinKarp creates a new RabinKarp value. The argument n defines the +// length of the byte sequence to be hashed. The default constant will will be +// used. +func NewRabinKarp(n int) *RabinKarp { + return NewRabinKarpConst(n, A) +} + +// NewRabinKarpConst creates a new RabinKarp value. The argument n defines the +// length of the byte sequence to be hashed. The argument a provides the +// constant used to compute the hash. +func NewRabinKarpConst(n int, a uint64) *RabinKarp { + if n <= 0 { + panic("number of bytes n must be positive") + } + aOldest := uint64(1) + // There are faster methods. For the small n required by the LZMA + // compressor O(n) is sufficient. + for i := 0; i < n; i++ { + aOldest *= a + } + return &RabinKarp{ + A: a, aOldest: aOldest, + p: make([]byte, 0, n), + } +} + +// Len returns the length of the byte sequence. +func (r *RabinKarp) Len() int { + return cap(r.p) +} + +// RollByte computes the hash after x has been added. +func (r *RabinKarp) RollByte(x byte) uint64 { + if len(r.p) < cap(r.p) { + r.h += uint64(x) + r.h *= r.A + r.p = append(r.p, x) + } else { + r.h -= uint64(r.p[r.i]) * r.aOldest + r.h += uint64(x) + r.h *= r.A + r.p[r.i] = x + r.i = (r.i + 1) % cap(r.p) + } + return r.h +} diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/roller.go b/vendor/github.com/ulikunitz/xz/internal/hash/roller.go new file mode 100644 index 0000000..ab6a19c --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/roller.go @@ -0,0 +1,29 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hash + +// Roller provides an interface for rolling hashes. The hash value will become +// valid after hash has been called Len times. +type Roller interface { + Len() int + RollByte(x byte) uint64 +} + +// Hashes computes all hash values for the array p. Note that the state of the +// roller is changed. +func Hashes(r Roller, p []byte) []uint64 { + n := r.Len() + if len(p) < n { + return nil + } + h := make([]uint64, len(p)-n+1) + for i := 0; i < n-1; i++ { + r.RollByte(p[i]) + } + for i := range h { + h[i] = r.RollByte(p[i+n-1]) + } + return h +} diff --git a/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go b/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go new file mode 100644 index 0000000..0ba45e8 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go @@ -0,0 +1,457 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xlog provides a simple logging package that allows to disable +// certain message categories. It defines a type, Logger, with multiple +// methods for formatting output. The package has also a predefined +// 'standard' Logger accessible through helper function Print[f|ln], +// Fatal[f|ln], Panic[f|ln], Warn[f|ln], Print[f|ln] and Debug[f|ln] +// that are easier to use then creating a Logger manually. That logger +// writes to standard error and prints the date and time of each logged +// message, which can be configured using the function SetFlags. +// +// The Fatal functions call os.Exit(1) after the message is output +// unless not suppressed by the flags. The Panic functions call panic +// after the writing the log message unless suppressed. +package xlog + +import ( + "fmt" + "io" + "os" + "runtime" + "sync" + "time" +) + +// The flags define what information is prefixed to each log entry +// generated by the Logger. The Lno* versions allow the suppression of +// specific output. The bits are or'ed together to control what will be +// printed. There is no control over the order of the items printed and +// the format. The full format is: +// +// 2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message +// +const ( + Ldate = 1 << iota // the date: 2009-01-23 + Ltime // the time: 01:23:23 + Lmicroseconds // microsecond resolution: 01:23:23.123123 + Llongfile // full file name and line number: /a/b/c/d.go:23 + Lshortfile // final file name element and line number: d.go:23 + Lnopanic // suppresses output from Panic[f|ln] but not the panic call + Lnofatal // suppresses output from Fatal[f|ln] but not the exit + Lnowarn // suppresses output from Warn[f|ln] + Lnoprint // suppresses output from Print[f|ln] + Lnodebug // suppresses output from Debug[f|ln] + // initial values for the standard logger + Lstdflags = Ldate | Ltime | Lnodebug +) + +// A Logger represents an active logging object that generates lines of +// output to an io.Writer. Each logging operation if not suppressed +// makes a single call to the Writer's Write method. A Logger can be +// used simultaneously from multiple goroutines; it guarantees to +// serialize access to the Writer. +type Logger struct { + mu sync.Mutex // ensures atomic writes; and protects the following + // fields + prefix string // prefix to write at beginning of each line + flag int // properties + out io.Writer // destination for output + buf []byte // for accumulating text to write +} + +// New creates a new Logger. The out argument sets the destination to +// which the log output will be written. The prefix appears at the +// beginning of each log line. The flag argument defines the logging +// properties. +func New(out io.Writer, prefix string, flag int) *Logger { + return &Logger{out: out, prefix: prefix, flag: flag} +} + +// std is the standard logger used by the package scope functions. +var std = New(os.Stderr, "", Lstdflags) + +// itoa converts the integer to ASCII. A negative widths will avoid +// zero-padding. The function supports only non-negative integers. +func itoa(buf *[]byte, i int, wid int) { + var u = uint(i) + if u == 0 && wid <= 1 { + *buf = append(*buf, '0') + return + } + var b [32]byte + bp := len(b) + for ; u > 0 || wid > 0; u /= 10 { + bp-- + wid-- + b[bp] = byte(u%10) + '0' + } + *buf = append(*buf, b[bp:]...) +} + +// formatHeader puts the header into the buf field of the buffer. +func (l *Logger) formatHeader(t time.Time, file string, line int) { + l.buf = append(l.buf, l.prefix...) + if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 { + if l.flag&Ldate != 0 { + year, month, day := t.Date() + itoa(&l.buf, year, 4) + l.buf = append(l.buf, '-') + itoa(&l.buf, int(month), 2) + l.buf = append(l.buf, '-') + itoa(&l.buf, day, 2) + l.buf = append(l.buf, ' ') + } + if l.flag&(Ltime|Lmicroseconds) != 0 { + hour, min, sec := t.Clock() + itoa(&l.buf, hour, 2) + l.buf = append(l.buf, ':') + itoa(&l.buf, min, 2) + l.buf = append(l.buf, ':') + itoa(&l.buf, sec, 2) + if l.flag&Lmicroseconds != 0 { + l.buf = append(l.buf, '.') + itoa(&l.buf, t.Nanosecond()/1e3, 6) + } + l.buf = append(l.buf, ' ') + } + } + if l.flag&(Lshortfile|Llongfile) != 0 { + if l.flag&Lshortfile != 0 { + short := file + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + short = file[i+1:] + break + } + } + file = short + } + l.buf = append(l.buf, file...) + l.buf = append(l.buf, ':') + itoa(&l.buf, line, -1) + l.buf = append(l.buf, ": "...) + } +} + +func (l *Logger) output(calldepth int, now time.Time, s string) error { + var file string + var line int + if l.flag&(Lshortfile|Llongfile) != 0 { + l.mu.Unlock() + var ok bool + _, file, line, ok = runtime.Caller(calldepth) + if !ok { + file = "???" + line = 0 + } + l.mu.Lock() + } + l.buf = l.buf[:0] + l.formatHeader(now, file, line) + l.buf = append(l.buf, s...) + if len(s) == 0 || s[len(s)-1] != '\n' { + l.buf = append(l.buf, '\n') + } + _, err := l.out.Write(l.buf) + return err +} + +// Output writes the string s with the header controlled by the flags to +// the l.out writer. A newline will be appended if s doesn't end in a +// newline. Calldepth is used to recover the PC, although all current +// calls of Output use the call depth 2. Access to the function is serialized. +func (l *Logger) Output(calldepth, noflag int, v ...interface{}) error { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.flag&noflag != 0 { + return nil + } + s := fmt.Sprint(v...) + return l.output(calldepth+1, now, s) +} + +// Outputf works like output but formats the output like Printf. +func (l *Logger) Outputf(calldepth int, noflag int, format string, v ...interface{}) error { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.flag&noflag != 0 { + return nil + } + s := fmt.Sprintf(format, v...) + return l.output(calldepth+1, now, s) +} + +// Outputln works like output but formats the output like Println. +func (l *Logger) Outputln(calldepth int, noflag int, v ...interface{}) error { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.flag&noflag != 0 { + return nil + } + s := fmt.Sprintln(v...) + return l.output(calldepth+1, now, s) +} + +// Panic prints the message like Print and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func (l *Logger) Panic(v ...interface{}) { + l.Output(2, Lnopanic, v...) + s := fmt.Sprint(v...) + panic(s) +} + +// Panic prints the message like Print and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func Panic(v ...interface{}) { + std.Output(2, Lnopanic, v...) + s := fmt.Sprint(v...) + panic(s) +} + +// Panicf prints the message like Printf and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func (l *Logger) Panicf(format string, v ...interface{}) { + l.Outputf(2, Lnopanic, format, v...) + s := fmt.Sprintf(format, v...) + panic(s) +} + +// Panicf prints the message like Printf and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func Panicf(format string, v ...interface{}) { + std.Outputf(2, Lnopanic, format, v...) + s := fmt.Sprintf(format, v...) + panic(s) +} + +// Panicln prints the message like Println and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func (l *Logger) Panicln(v ...interface{}) { + l.Outputln(2, Lnopanic, v...) + s := fmt.Sprintln(v...) + panic(s) +} + +// Panicln prints the message like Println and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func Panicln(v ...interface{}) { + std.Outputln(2, Lnopanic, v...) + s := fmt.Sprintln(v...) + panic(s) +} + +// Fatal prints the message like Print and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func (l *Logger) Fatal(v ...interface{}) { + l.Output(2, Lnofatal, v...) + os.Exit(1) +} + +// Fatal prints the message like Print and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func Fatal(v ...interface{}) { + std.Output(2, Lnofatal, v...) + os.Exit(1) +} + +// Fatalf prints the message like Printf and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func (l *Logger) Fatalf(format string, v ...interface{}) { + l.Outputf(2, Lnofatal, format, v...) + os.Exit(1) +} + +// Fatalf prints the message like Printf and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func Fatalf(format string, v ...interface{}) { + std.Outputf(2, Lnofatal, format, v...) + os.Exit(1) +} + +// Fatalln prints the message like Println and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func (l *Logger) Fatalln(format string, v ...interface{}) { + l.Outputln(2, Lnofatal, v...) + os.Exit(1) +} + +// Fatalln prints the message like Println and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func Fatalln(format string, v ...interface{}) { + std.Outputln(2, Lnofatal, v...) + os.Exit(1) +} + +// Warn prints the message like Print. The printing might be suppressed +// by the flag Lnowarn. +func (l *Logger) Warn(v ...interface{}) { + l.Output(2, Lnowarn, v...) +} + +// Warn prints the message like Print. The printing might be suppressed +// by the flag Lnowarn. +func Warn(v ...interface{}) { + std.Output(2, Lnowarn, v...) +} + +// Warnf prints the message like Printf. The printing might be suppressed +// by the flag Lnowarn. +func (l *Logger) Warnf(format string, v ...interface{}) { + l.Outputf(2, Lnowarn, format, v...) +} + +// Warnf prints the message like Printf. The printing might be suppressed +// by the flag Lnowarn. +func Warnf(format string, v ...interface{}) { + std.Outputf(2, Lnowarn, format, v...) +} + +// Warnln prints the message like Println. The printing might be suppressed +// by the flag Lnowarn. +func (l *Logger) Warnln(v ...interface{}) { + l.Outputln(2, Lnowarn, v...) +} + +// Warnln prints the message like Println. The printing might be suppressed +// by the flag Lnowarn. +func Warnln(v ...interface{}) { + std.Outputln(2, Lnowarn, v...) +} + +// Print prints the message like fmt.Print. The printing might be suppressed +// by the flag Lnoprint. +func (l *Logger) Print(v ...interface{}) { + l.Output(2, Lnoprint, v...) +} + +// Print prints the message like fmt.Print. The printing might be suppressed +// by the flag Lnoprint. +func Print(v ...interface{}) { + std.Output(2, Lnoprint, v...) +} + +// Printf prints the message like fmt.Printf. The printing might be suppressed +// by the flag Lnoprint. +func (l *Logger) Printf(format string, v ...interface{}) { + l.Outputf(2, Lnoprint, format, v...) +} + +// Printf prints the message like fmt.Printf. The printing might be suppressed +// by the flag Lnoprint. +func Printf(format string, v ...interface{}) { + std.Outputf(2, Lnoprint, format, v...) +} + +// Println prints the message like fmt.Println. The printing might be +// suppressed by the flag Lnoprint. +func (l *Logger) Println(v ...interface{}) { + l.Outputln(2, Lnoprint, v...) +} + +// Println prints the message like fmt.Println. The printing might be +// suppressed by the flag Lnoprint. +func Println(v ...interface{}) { + std.Outputln(2, Lnoprint, v...) +} + +// Debug prints the message like Print. The printing might be suppressed +// by the flag Lnodebug. +func (l *Logger) Debug(v ...interface{}) { + l.Output(2, Lnodebug, v...) +} + +// Debug prints the message like Print. The printing might be suppressed +// by the flag Lnodebug. +func Debug(v ...interface{}) { + std.Output(2, Lnodebug, v...) +} + +// Debugf prints the message like Printf. The printing might be suppressed +// by the flag Lnodebug. +func (l *Logger) Debugf(format string, v ...interface{}) { + l.Outputf(2, Lnodebug, format, v...) +} + +// Debugf prints the message like Printf. The printing might be suppressed +// by the flag Lnodebug. +func Debugf(format string, v ...interface{}) { + std.Outputf(2, Lnodebug, format, v...) +} + +// Debugln prints the message like Println. The printing might be suppressed +// by the flag Lnodebug. +func (l *Logger) Debugln(v ...interface{}) { + l.Outputln(2, Lnodebug, v...) +} + +// Debugln prints the message like Println. The printing might be suppressed +// by the flag Lnodebug. +func Debugln(v ...interface{}) { + std.Outputln(2, Lnodebug, v...) +} + +// Flags returns the current flags used by the logger. +func (l *Logger) Flags() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.flag +} + +// Flags returns the current flags used by the standard logger. +func Flags() int { + return std.Flags() +} + +// SetFlags sets the flags of the logger. +func (l *Logger) SetFlags(flag int) { + l.mu.Lock() + defer l.mu.Unlock() + l.flag = flag +} + +// SetFlags sets the flags for the standard logger. +func SetFlags(flag int) { + std.SetFlags(flag) +} + +// Prefix returns the prefix used by the logger. +func (l *Logger) Prefix() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.prefix +} + +// Prefix returns the prefix used by the standard logger of the package. +func Prefix() string { + return std.Prefix() +} + +// SetPrefix sets the prefix for the logger. +func (l *Logger) SetPrefix(prefix string) { + l.mu.Lock() + defer l.mu.Unlock() + l.prefix = prefix +} + +// SetPrefix sets the prefix of the standard logger of the package. +func SetPrefix(prefix string) { + std.SetPrefix(prefix) +} + +// SetOutput sets the output of the logger. +func (l *Logger) SetOutput(w io.Writer) { + l.mu.Lock() + defer l.mu.Unlock() + l.out = w +} + +// SetOutput sets the output for the standard logger of the package. +func SetOutput(w io.Writer) { + std.SetOutput(w) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bintree.go b/vendor/github.com/ulikunitz/xz/lzma/bintree.go new file mode 100644 index 0000000..a781bd1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/bintree.go @@ -0,0 +1,523 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "bufio" + "errors" + "fmt" + "io" + "unicode" +) + +// node represents a node in the binary tree. +type node struct { + // x is the search value + x uint32 + // p parent node + p uint32 + // l left child + l uint32 + // r right child + r uint32 +} + +// wordLen is the number of bytes represented by the v field of a node. +const wordLen = 4 + +// binTree supports the identification of the next operation based on a +// binary tree. +// +// Nodes will be identified by their index into the ring buffer. +type binTree struct { + dict *encoderDict + // ring buffer of nodes + node []node + // absolute offset of the entry for the next node. Position 4 + // byte larger. + hoff int64 + // front position in the node ring buffer + front uint32 + // index of the root node + root uint32 + // current x value + x uint32 + // preallocated array + data []byte +} + +// null represents the nonexistent index. We can't use zero because it +// would always exist or we would need to decrease the index for each +// reference. +const null uint32 = 1<<32 - 1 + +// newBinTree initializes the binTree structure. The capacity defines +// the size of the buffer and defines the maximum distance for which +// matches will be found. +func newBinTree(capacity int) (t *binTree, err error) { + if capacity < 1 { + return nil, errors.New( + "newBinTree: capacity must be larger than zero") + } + if int64(capacity) >= int64(null) { + return nil, errors.New( + "newBinTree: capacity must less 2^{32}-1") + } + t = &binTree{ + node: make([]node, capacity), + hoff: -int64(wordLen), + root: null, + data: make([]byte, maxMatchLen), + } + return t, nil +} + +func (t *binTree) SetDict(d *encoderDict) { t.dict = d } + +// WriteByte writes a single byte into the binary tree. +func (t *binTree) WriteByte(c byte) error { + t.x = (t.x << 8) | uint32(c) + t.hoff++ + if t.hoff < 0 { + return nil + } + v := t.front + if int64(v) < t.hoff { + // We are overwriting old nodes stored in the tree. + t.remove(v) + } + t.node[v].x = t.x + t.add(v) + t.front++ + if int64(t.front) >= int64(len(t.node)) { + t.front = 0 + } + return nil +} + +// Writes writes a sequence of bytes into the binTree structure. +func (t *binTree) Write(p []byte) (n int, err error) { + for _, c := range p { + t.WriteByte(c) + } + return len(p), nil +} + +// add puts the node v into the tree. The node must not be part of the +// tree before. +func (t *binTree) add(v uint32) { + vn := &t.node[v] + // Set left and right to null indices. + vn.l, vn.r = null, null + // If the binary tree is empty make v the root. + if t.root == null { + t.root = v + vn.p = null + return + } + x := vn.x + p := t.root + // Search for the right leave link and add the new node. + for { + pn := &t.node[p] + if x <= pn.x { + if pn.l == null { + pn.l = v + vn.p = p + return + } + p = pn.l + } else { + if pn.r == null { + pn.r = v + vn.p = p + return + } + p = pn.r + } + } +} + +// parent returns the parent node index of v and the pointer to v value +// in the parent. +func (t *binTree) parent(v uint32) (p uint32, ptr *uint32) { + if t.root == v { + return null, &t.root + } + p = t.node[v].p + if t.node[p].l == v { + ptr = &t.node[p].l + } else { + ptr = &t.node[p].r + } + return +} + +// Remove node v. +func (t *binTree) remove(v uint32) { + vn := &t.node[v] + p, ptr := t.parent(v) + l, r := vn.l, vn.r + if l == null { + // Move the right child up. + *ptr = r + if r != null { + t.node[r].p = p + } + return + } + if r == null { + // Move the left child up. + *ptr = l + t.node[l].p = p + return + } + + // Search the in-order predecessor u. + un := &t.node[l] + ur := un.r + if ur == null { + // In order predecessor is l. Move it up. + un.r = r + t.node[r].p = l + un.p = p + *ptr = l + return + } + var u uint32 + for { + // Look for the max value in the tree where l is root. + u = ur + ur = t.node[u].r + if ur == null { + break + } + } + // replace u with ul + un = &t.node[u] + ul := un.l + up := un.p + t.node[up].r = ul + if ul != null { + t.node[ul].p = up + } + + // replace v by u + un.l, un.r = l, r + t.node[l].p = u + t.node[r].p = u + *ptr = u + un.p = p +} + +// search looks for the node that have the value x or for the nodes that +// brace it. The node highest in the tree with the value x will be +// returned. All other nodes with the same value live in left subtree of +// the returned node. +func (t *binTree) search(v uint32, x uint32) (a, b uint32) { + a, b = null, null + if v == null { + return + } + for { + vn := &t.node[v] + if x <= vn.x { + if x == vn.x { + return v, v + } + b = v + if vn.l == null { + return + } + v = vn.l + } else { + a = v + if vn.r == null { + return + } + v = vn.r + } + } +} + +// max returns the node with maximum value in the subtree with v as +// root. +func (t *binTree) max(v uint32) uint32 { + if v == null { + return null + } + for { + r := t.node[v].r + if r == null { + return v + } + v = r + } +} + +// min returns the node with the minimum value in the subtree with v as +// root. +func (t *binTree) min(v uint32) uint32 { + if v == null { + return null + } + for { + l := t.node[v].l + if l == null { + return v + } + v = l + } +} + +// pred returns the in-order predecessor of node v. +func (t *binTree) pred(v uint32) uint32 { + if v == null { + return null + } + u := t.max(t.node[v].l) + if u != null { + return u + } + for { + p := t.node[v].p + if p == null { + return null + } + if t.node[p].r == v { + return p + } + v = p + } +} + +// succ returns the in-order successor of node v. +func (t *binTree) succ(v uint32) uint32 { + if v == null { + return null + } + u := t.min(t.node[v].r) + if u != null { + return u + } + for { + p := t.node[v].p + if p == null { + return null + } + if t.node[p].l == v { + return p + } + v = p + } +} + +// xval converts the first four bytes of a into an 32-bit unsigned +// integer in big-endian order. +func xval(a []byte) uint32 { + var x uint32 + switch len(a) { + default: + x |= uint32(a[3]) + fallthrough + case 3: + x |= uint32(a[2]) << 8 + fallthrough + case 2: + x |= uint32(a[1]) << 16 + fallthrough + case 1: + x |= uint32(a[0]) << 24 + case 0: + } + return x +} + +// dumpX converts value x into a four-letter string. +func dumpX(x uint32) string { + a := make([]byte, 4) + for i := 0; i < 4; i++ { + c := byte(x >> uint((3-i)*8)) + if unicode.IsGraphic(rune(c)) { + a[i] = c + } else { + a[i] = '.' + } + } + return string(a) +} + +// dumpNode writes a representation of the node v into the io.Writer. +func (t *binTree) dumpNode(w io.Writer, v uint32, indent int) { + if v == null { + return + } + + vn := &t.node[v] + + t.dumpNode(w, vn.r, indent+2) + + for i := 0; i < indent; i++ { + fmt.Fprint(w, " ") + } + if vn.p == null { + fmt.Fprintf(w, "node %d %q parent null\n", v, dumpX(vn.x)) + } else { + fmt.Fprintf(w, "node %d %q parent %d\n", v, dumpX(vn.x), vn.p) + } + + t.dumpNode(w, vn.l, indent+2) +} + +// dump prints a representation of the binary tree into the writer. +func (t *binTree) dump(w io.Writer) error { + bw := bufio.NewWriter(w) + t.dumpNode(bw, t.root, 0) + return bw.Flush() +} + +func (t *binTree) distance(v uint32) int { + dist := int(t.front) - int(v) + if dist <= 0 { + dist += len(t.node) + } + return dist +} + +type matchParams struct { + rep [4]uint32 + // length when match will be accepted + nAccept int + // nodes to check + check int + // finish if length get shorter + stopShorter bool +} + +func (t *binTree) match(m match, distIter func() (int, bool), p matchParams, +) (r match, checked int, accepted bool) { + buf := &t.dict.buf + for { + if checked >= p.check { + return m, checked, true + } + dist, ok := distIter() + if !ok { + return m, checked, false + } + checked++ + if m.n > 0 { + i := buf.rear - dist + m.n - 1 + if i < 0 { + i += len(buf.data) + } else if i >= len(buf.data) { + i -= len(buf.data) + } + if buf.data[i] != t.data[m.n-1] { + if p.stopShorter { + return m, checked, false + } + continue + } + } + n := buf.matchLen(dist, t.data) + switch n { + case 0: + if p.stopShorter { + return m, checked, false + } + continue + case 1: + if uint32(dist-minDistance) != p.rep[0] { + continue + } + } + if n < m.n || (n == m.n && int64(dist) >= m.distance) { + continue + } + m = match{int64(dist), n} + if n >= p.nAccept { + return m, checked, true + } + } +} + +func (t *binTree) NextOp(rep [4]uint32) operation { + // retrieve maxMatchLen data + n, _ := t.dict.buf.Peek(t.data[:maxMatchLen]) + if n == 0 { + panic("no data in buffer") + } + t.data = t.data[:n] + + var ( + m match + x, u, v uint32 + iterPred, iterSucc func() (int, bool) + ) + p := matchParams{ + rep: rep, + nAccept: maxMatchLen, + check: 32, + } + i := 4 + iterSmall := func() (dist int, ok bool) { + i-- + if i <= 0 { + return 0, false + } + return i, true + } + m, checked, accepted := t.match(m, iterSmall, p) + if accepted { + goto end + } + p.check -= checked + x = xval(t.data) + u, v = t.search(t.root, x) + if u == v && len(t.data) == 4 { + iter := func() (dist int, ok bool) { + if u == null { + return 0, false + } + dist = t.distance(u) + u, v = t.search(t.node[u].l, x) + if u != v { + u = null + } + return dist, true + } + m, _, _ = t.match(m, iter, p) + goto end + } + p.stopShorter = true + iterSucc = func() (dist int, ok bool) { + if v == null { + return 0, false + } + dist = t.distance(v) + v = t.succ(v) + return dist, true + } + m, checked, accepted = t.match(m, iterSucc, p) + if accepted { + goto end + } + p.check -= checked + iterPred = func() (dist int, ok bool) { + if u == null { + return 0, false + } + dist = t.distance(u) + u = t.pred(u) + return dist, true + } + m, _, _ = t.match(m, iterPred, p) +end: + if m.n == 0 { + return lit{t.data[0]} + } + return m +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bitops.go b/vendor/github.com/ulikunitz/xz/lzma/bitops.go new file mode 100644 index 0000000..e9bab01 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/bitops.go @@ -0,0 +1,45 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +/* Naming conventions follows the CodeReviewComments in the Go Wiki. */ + +// ntz32Const is used by the functions NTZ and NLZ. +const ntz32Const = 0x04d7651f + +// ntz32Table is a helper table for de Bruijn algorithm by Danny Dubé. +// See Henry S. Warren, Jr. "Hacker's Delight" section 5-1 figure 5-26. +var ntz32Table = [32]int8{ + 0, 1, 2, 24, 3, 19, 6, 25, + 22, 4, 20, 10, 16, 7, 12, 26, + 31, 23, 18, 5, 21, 9, 15, 11, + 30, 17, 8, 14, 29, 13, 28, 27, +} + +// ntz32 computes the number of trailing zeros for an unsigned 32-bit integer. +func ntz32(x uint32) int { + if x == 0 { + return 32 + } + x = (x & -x) * ntz32Const + return int(ntz32Table[x>>27]) +} + +// nlz32 computes the number of leading zeros for an unsigned 32-bit integer. +func nlz32(x uint32) int { + // Smear left most bit to the right + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + // Use ntz mechanism to calculate nlz. + x++ + if x == 0 { + return 0 + } + x *= ntz32Const + return 32 - int(ntz32Table[x>>27]) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/breader.go b/vendor/github.com/ulikunitz/xz/lzma/breader.go new file mode 100644 index 0000000..5350d81 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/breader.go @@ -0,0 +1,39 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" +) + +// breader provides the ReadByte function for a Reader. It doesn't read +// more data from the reader than absolutely necessary. +type breader struct { + io.Reader + // helper slice to save allocations + p []byte +} + +// ByteReader converts an io.Reader into an io.ByteReader. +func ByteReader(r io.Reader) io.ByteReader { + br, ok := r.(io.ByteReader) + if !ok { + return &breader{r, make([]byte, 1)} + } + return br +} + +// ReadByte read byte function. +func (r *breader) ReadByte() (c byte, err error) { + n, err := r.Reader.Read(r.p) + if n < 1 { + if err == nil { + err = errors.New("breader.ReadByte: no data") + } + return 0, err + } + return r.p[0], nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/buffer.go b/vendor/github.com/ulikunitz/xz/lzma/buffer.go new file mode 100644 index 0000000..50e0b6d --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/buffer.go @@ -0,0 +1,171 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" +) + +// buffer provides a circular buffer of bytes. If the front index equals +// the rear index the buffer is empty. As a consequence front cannot be +// equal rear for a full buffer. So a full buffer has a length that is +// one byte less the the length of the data slice. +type buffer struct { + data []byte + front int + rear int +} + +// newBuffer creates a buffer with the given size. +func newBuffer(size int) *buffer { + return &buffer{data: make([]byte, size+1)} +} + +// Cap returns the capacity of the buffer. +func (b *buffer) Cap() int { + return len(b.data) - 1 +} + +// Resets the buffer. The front and rear index are set to zero. +func (b *buffer) Reset() { + b.front = 0 + b.rear = 0 +} + +// Buffered returns the number of bytes buffered. +func (b *buffer) Buffered() int { + delta := b.front - b.rear + if delta < 0 { + delta += len(b.data) + } + return delta +} + +// Available returns the number of bytes available for writing. +func (b *buffer) Available() int { + delta := b.rear - 1 - b.front + if delta < 0 { + delta += len(b.data) + } + return delta +} + +// addIndex adds a non-negative integer to the index i and returns the +// resulting index. The function takes care of wrapping the index as +// well as potential overflow situations. +func (b *buffer) addIndex(i int, n int) int { + // subtraction of len(b.data) prevents overflow + i += n - len(b.data) + if i < 0 { + i += len(b.data) + } + return i +} + +// Read reads bytes from the buffer into p and returns the number of +// bytes read. The function never returns an error but might return less +// data than requested. +func (b *buffer) Read(p []byte) (n int, err error) { + n, err = b.Peek(p) + b.rear = b.addIndex(b.rear, n) + return n, err +} + +// Peek reads bytes from the buffer into p without changing the buffer. +// Peek will never return an error but might return less data than +// requested. +func (b *buffer) Peek(p []byte) (n int, err error) { + m := b.Buffered() + n = len(p) + if m < n { + n = m + p = p[:n] + } + k := copy(p, b.data[b.rear:]) + if k < n { + copy(p[k:], b.data) + } + return n, nil +} + +// Discard skips the n next bytes to read from the buffer, returning the +// bytes discarded. +// +// If Discards skips fewer than n bytes, it returns an error. +func (b *buffer) Discard(n int) (discarded int, err error) { + if n < 0 { + return 0, errors.New("buffer.Discard: negative argument") + } + m := b.Buffered() + if m < n { + n = m + err = errors.New( + "buffer.Discard: discarded less bytes then requested") + } + b.rear = b.addIndex(b.rear, n) + return n, err +} + +// ErrNoSpace indicates that there is insufficient space for the Write +// operation. +var ErrNoSpace = errors.New("insufficient space") + +// Write puts data into the buffer. If less bytes are written than +// requested ErrNoSpace is returned. +func (b *buffer) Write(p []byte) (n int, err error) { + m := b.Available() + n = len(p) + if m < n { + n = m + p = p[:m] + err = ErrNoSpace + } + k := copy(b.data[b.front:], p) + if k < n { + copy(b.data, p[k:]) + } + b.front = b.addIndex(b.front, n) + return n, err +} + +// WriteByte writes a single byte into the buffer. The error ErrNoSpace +// is returned if no single byte is available in the buffer for writing. +func (b *buffer) WriteByte(c byte) error { + if b.Available() < 1 { + return ErrNoSpace + } + b.data[b.front] = c + b.front = b.addIndex(b.front, 1) + return nil +} + +// prefixLen returns the length of the common prefix of a and b. +func prefixLen(a, b []byte) int { + if len(a) > len(b) { + a, b = b, a + } + for i, c := range a { + if b[i] != c { + return i + } + } + return len(a) +} + +// matchLen returns the length of the common prefix for the given +// distance from the rear and the byte slice p. +func (b *buffer) matchLen(distance int, p []byte) int { + var n int + i := b.rear - distance + if i < 0 { + if n = prefixLen(p, b.data[len(b.data)+i:]); n < -i { + return n + } + p = p[n:] + i = 0 + } + n += prefixLen(p, b.data[i:]) + return n +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go b/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go new file mode 100644 index 0000000..a3696ba --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go @@ -0,0 +1,37 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" +) + +// ErrLimit indicates that the limit of the LimitedByteWriter has been +// reached. +var ErrLimit = errors.New("limit reached") + +// LimitedByteWriter provides a byte writer that can be written until a +// limit is reached. The field N provides the number of remaining +// bytes. +type LimitedByteWriter struct { + BW io.ByteWriter + N int64 +} + +// WriteByte writes a single byte to the limited byte writer. It returns +// ErrLimit if the limit has been reached. If the byte is successfully +// written the field N of the LimitedByteWriter will be decremented by +// one. +func (l *LimitedByteWriter) WriteByte(c byte) error { + if l.N <= 0 { + return ErrLimit + } + if err := l.BW.WriteByte(c); err != nil { + return err + } + l.N-- + return nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoder.go b/vendor/github.com/ulikunitz/xz/lzma/decoder.go new file mode 100644 index 0000000..16e14db --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/decoder.go @@ -0,0 +1,277 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "io" +) + +// decoder decodes a raw LZMA stream without any header. +type decoder struct { + // dictionary; the rear pointer of the buffer will be used for + // reading the data. + Dict *decoderDict + // decoder state + State *state + // range decoder + rd *rangeDecoder + // start stores the head value of the dictionary for the LZMA + // stream + start int64 + // size of uncompressed data + size int64 + // end-of-stream encountered + eos bool + // EOS marker found + eosMarker bool +} + +// newDecoder creates a new decoder instance. The parameter size provides +// the expected byte size of the decompressed data. If the size is +// unknown use a negative value. In that case the decoder will look for +// a terminating end-of-stream marker. +func newDecoder(br io.ByteReader, state *state, dict *decoderDict, size int64) (d *decoder, err error) { + rd, err := newRangeDecoder(br) + if err != nil { + return nil, err + } + d = &decoder{ + State: state, + Dict: dict, + rd: rd, + size: size, + start: dict.pos(), + } + return d, nil +} + +// Reopen restarts the decoder with a new byte reader and a new size. Reopen +// resets the Decompressed counter to zero. +func (d *decoder) Reopen(br io.ByteReader, size int64) error { + var err error + if d.rd, err = newRangeDecoder(br); err != nil { + return err + } + d.start = d.Dict.pos() + d.size = size + d.eos = false + return nil +} + +// decodeLiteral decodes a single literal from the LZMA stream. +func (d *decoder) decodeLiteral() (op operation, err error) { + litState := d.State.litState(d.Dict.byteAt(1), d.Dict.head) + match := d.Dict.byteAt(int(d.State.rep[0]) + 1) + s, err := d.State.litCodec.Decode(d.rd, d.State.state, match, litState) + if err != nil { + return nil, err + } + return lit{s}, nil +} + +// errEOS indicates that an EOS marker has been found. +var errEOS = errors.New("EOS marker found") + +// readOp decodes the next operation from the compressed stream. It +// returns the operation. If an explicit end of stream marker is +// identified the eos error is returned. +func (d *decoder) readOp() (op operation, err error) { + // Value of the end of stream (EOS) marker + const eosDist = 1<<32 - 1 + + state, state2, posState := d.State.states(d.Dict.head) + + b, err := d.State.isMatch[state2].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + // literal + op, err := d.decodeLiteral() + if err != nil { + return nil, err + } + d.State.updateStateLiteral() + return op, nil + } + b, err = d.State.isRep[state].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + // simple match + d.State.rep[3], d.State.rep[2], d.State.rep[1] = + d.State.rep[2], d.State.rep[1], d.State.rep[0] + + d.State.updateStateMatch() + // The length decoder returns the length offset. + n, err := d.State.lenCodec.Decode(d.rd, posState) + if err != nil { + return nil, err + } + // The dist decoder returns the distance offset. The actual + // distance is 1 higher. + d.State.rep[0], err = d.State.distCodec.Decode(d.rd, n) + if err != nil { + return nil, err + } + if d.State.rep[0] == eosDist { + d.eosMarker = true + return nil, errEOS + } + op = match{n: int(n) + minMatchLen, + distance: int64(d.State.rep[0]) + minDistance} + return op, nil + } + b, err = d.State.isRepG0[state].Decode(d.rd) + if err != nil { + return nil, err + } + dist := d.State.rep[0] + if b == 0 { + // rep match 0 + b, err = d.State.isRepG0Long[state2].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + d.State.updateStateShortRep() + op = match{n: 1, distance: int64(dist) + minDistance} + return op, nil + } + } else { + b, err = d.State.isRepG1[state].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + dist = d.State.rep[1] + } else { + b, err = d.State.isRepG2[state].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + dist = d.State.rep[2] + } else { + dist = d.State.rep[3] + d.State.rep[3] = d.State.rep[2] + } + d.State.rep[2] = d.State.rep[1] + } + d.State.rep[1] = d.State.rep[0] + d.State.rep[0] = dist + } + n, err := d.State.repLenCodec.Decode(d.rd, posState) + if err != nil { + return nil, err + } + d.State.updateStateRep() + op = match{n: int(n) + minMatchLen, distance: int64(dist) + minDistance} + return op, nil +} + +// apply takes the operation and transforms the decoder dictionary accordingly. +func (d *decoder) apply(op operation) error { + var err error + switch x := op.(type) { + case match: + err = d.Dict.writeMatch(x.distance, x.n) + case lit: + err = d.Dict.WriteByte(x.b) + default: + panic("op is neither a match nor a literal") + } + return err +} + +// decompress fills the dictionary unless no space for new data is +// available. If the end of the LZMA stream has been reached io.EOF will +// be returned. +func (d *decoder) decompress() error { + if d.eos { + return io.EOF + } + for d.Dict.Available() >= maxMatchLen { + op, err := d.readOp() + switch err { + case nil: + break + case errEOS: + d.eos = true + if !d.rd.possiblyAtEnd() { + return errDataAfterEOS + } + if d.size >= 0 && d.size != d.Decompressed() { + return errSize + } + return io.EOF + case io.EOF: + d.eos = true + return io.ErrUnexpectedEOF + default: + return err + } + if err = d.apply(op); err != nil { + return err + } + if d.size >= 0 && d.Decompressed() >= d.size { + d.eos = true + if d.Decompressed() > d.size { + return errSize + } + if !d.rd.possiblyAtEnd() { + switch _, err = d.readOp(); err { + case nil: + return errSize + case io.EOF: + return io.ErrUnexpectedEOF + case errEOS: + break + default: + return err + } + } + return io.EOF + } + } + return nil +} + +// Errors that may be returned while decoding data. +var ( + errDataAfterEOS = errors.New("lzma: data after end of stream marker") + errSize = errors.New("lzma: wrong uncompressed data size") +) + +// Read reads data from the buffer. If no more data is available io.EOF is +// returned. +func (d *decoder) Read(p []byte) (n int, err error) { + var k int + for { + // Read of decoder dict never returns an error. + k, err = d.Dict.Read(p[n:]) + if err != nil { + panic(fmt.Errorf("dictionary read error %s", err)) + } + if k == 0 && d.eos { + return n, io.EOF + } + n += k + if n >= len(p) { + return n, nil + } + if err = d.decompress(); err != nil && err != io.EOF { + return n, err + } + } +} + +// Decompressed returns the number of bytes decompressed by the decoder. +func (d *decoder) Decompressed() int64 { + return d.Dict.pos() - d.start +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go new file mode 100644 index 0000000..564a12b --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go @@ -0,0 +1,135 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" +) + +// decoderDict provides the dictionary for the decoder. The whole +// dictionary is used as reader buffer. +type decoderDict struct { + buf buffer + head int64 +} + +// newDecoderDict creates a new decoder dictionary. The whole dictionary +// will be used as reader buffer. +func newDecoderDict(dictCap int) (d *decoderDict, err error) { + // lower limit supports easy test cases + if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) { + return nil, errors.New("lzma: dictCap out of range") + } + d = &decoderDict{buf: *newBuffer(dictCap)} + return d, nil +} + +// Reset clears the dictionary. The read buffer is not changed, so the +// buffered data can still be read. +func (d *decoderDict) Reset() { + d.head = 0 +} + +// WriteByte writes a single byte into the dictionary. It is used to +// write literals into the dictionary. +func (d *decoderDict) WriteByte(c byte) error { + if err := d.buf.WriteByte(c); err != nil { + return err + } + d.head++ + return nil +} + +// pos returns the position of the dictionary head. +func (d *decoderDict) pos() int64 { return d.head } + +// dictLen returns the actual length of the dictionary. +func (d *decoderDict) dictLen() int { + capacity := d.buf.Cap() + if d.head >= int64(capacity) { + return capacity + } + return int(d.head) +} + +// byteAt returns a byte stored in the dictionary. If the distance is +// non-positive or exceeds the current length of the dictionary the zero +// byte is returned. +func (d *decoderDict) byteAt(dist int) byte { + if !(0 < dist && dist <= d.dictLen()) { + return 0 + } + i := d.buf.front - dist + if i < 0 { + i += len(d.buf.data) + } + return d.buf.data[i] +} + +// writeMatch writes the match at the top of the dictionary. The given +// distance must point in the current dictionary and the length must not +// exceed the maximum length 273 supported in LZMA. +// +// The error value ErrNoSpace indicates that no space is available in +// the dictionary for writing. You need to read from the dictionary +// first. +func (d *decoderDict) writeMatch(dist int64, length int) error { + if !(0 < dist && dist <= int64(d.dictLen())) { + return errors.New("writeMatch: distance out of range") + } + if !(0 < length && length <= maxMatchLen) { + return errors.New("writeMatch: length out of range") + } + if length > d.buf.Available() { + return ErrNoSpace + } + d.head += int64(length) + + i := d.buf.front - int(dist) + if i < 0 { + i += len(d.buf.data) + } + for length > 0 { + var p []byte + if i >= d.buf.front { + p = d.buf.data[i:] + i = 0 + } else { + p = d.buf.data[i:d.buf.front] + i = d.buf.front + } + if len(p) > length { + p = p[:length] + } + if _, err := d.buf.Write(p); err != nil { + panic(fmt.Errorf("d.buf.Write returned error %s", err)) + } + length -= len(p) + } + return nil +} + +// Write writes the given bytes into the dictionary and advances the +// head. +func (d *decoderDict) Write(p []byte) (n int, err error) { + n, err = d.buf.Write(p) + d.head += int64(n) + return n, err +} + +// Available returns the number of available bytes for writing into the +// decoder dictionary. +func (d *decoderDict) Available() int { return d.buf.Available() } + +// Read reads data from the buffer contained in the decoder dictionary. +func (d *decoderDict) Read(p []byte) (n int, err error) { return d.buf.Read(p) } + +// Buffered returns the number of bytes currently buffered in the +// decoder dictionary. +func (d *decoderDict) buffered() int { return d.buf.Buffered() } + +// Peek gets data from the buffer without advancing the rear index. +func (d *decoderDict) peek(p []byte) (n int, err error) { return d.buf.Peek(p) } diff --git a/vendor/github.com/ulikunitz/xz/lzma/directcodec.go b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go new file mode 100644 index 0000000..e08eb98 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go @@ -0,0 +1,49 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import "fmt" + +// directCodec allows the encoding and decoding of values with a fixed number +// of bits. The number of bits must be in the range [1,32]. +type directCodec byte + +// makeDirectCodec creates a directCodec. The function panics if the number of +// bits is not in the range [1,32]. +func makeDirectCodec(bits int) directCodec { + if !(1 <= bits && bits <= 32) { + panic(fmt.Errorf("bits=%d out of range", bits)) + } + return directCodec(bits) +} + +// Bits returns the number of bits supported by this codec. +func (dc directCodec) Bits() int { + return int(dc) +} + +// Encode uses the range encoder to encode a value with the fixed number of +// bits. The most-significant bit is encoded first. +func (dc directCodec) Encode(e *rangeEncoder, v uint32) error { + for i := int(dc) - 1; i >= 0; i-- { + if err := e.DirectEncodeBit(v >> uint(i)); err != nil { + return err + } + } + return nil +} + +// Decode uses the range decoder to decode a value with the given number of +// given bits. The most-significant bit is decoded first. +func (dc directCodec) Decode(d *rangeDecoder) (v uint32, err error) { + for i := int(dc) - 1; i >= 0; i-- { + x, err := d.DirectDecodeBit() + if err != nil { + return 0, err + } + v = (v << 1) | x + } + return v, nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/distcodec.go b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go new file mode 100644 index 0000000..b053a2d --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go @@ -0,0 +1,156 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// Constants used by the distance codec. +const ( + // minimum supported distance + minDistance = 1 + // maximum supported distance, value is used for the eos marker. + maxDistance = 1 << 32 + // number of the supported len states + lenStates = 4 + // start for the position models + startPosModel = 4 + // first index with align bits support + endPosModel = 14 + // bits for the position slots + posSlotBits = 6 + // number of align bits + alignBits = 4 + // maximum position slot + maxPosSlot = 63 +) + +// distCodec provides encoding and decoding of distance values. +type distCodec struct { + posSlotCodecs [lenStates]treeCodec + posModel [endPosModel - startPosModel]treeReverseCodec + alignCodec treeReverseCodec +} + +// deepcopy initializes dc as deep copy of the source. +func (dc *distCodec) deepcopy(src *distCodec) { + if dc == src { + return + } + for i := range dc.posSlotCodecs { + dc.posSlotCodecs[i].deepcopy(&src.posSlotCodecs[i]) + } + for i := range dc.posModel { + dc.posModel[i].deepcopy(&src.posModel[i]) + } + dc.alignCodec.deepcopy(&src.alignCodec) +} + +// distBits returns the number of bits required to encode dist. +func distBits(dist uint32) int { + if dist < startPosModel { + return 6 + } + // slot s > 3, dist d + // s = 2(bits(d)-1) + bit(d, bits(d)-2) + // s>>1 = bits(d)-1 + // bits(d) = 32-nlz32(d) + // s>>1=31-nlz32(d) + // n = 5 + (s>>1) = 36 - nlz32(d) + return 36 - nlz32(dist) +} + +// newDistCodec creates a new distance codec. +func (dc *distCodec) init() { + for i := range dc.posSlotCodecs { + dc.posSlotCodecs[i] = makeTreeCodec(posSlotBits) + } + for i := range dc.posModel { + posSlot := startPosModel + i + bits := (posSlot >> 1) - 1 + dc.posModel[i] = makeTreeReverseCodec(bits) + } + dc.alignCodec = makeTreeReverseCodec(alignBits) +} + +// lenState converts the value l to a supported lenState value. +func lenState(l uint32) uint32 { + if l >= lenStates { + l = lenStates - 1 + } + return l +} + +// Encode encodes the distance using the parameter l. Dist can have values from +// the full range of uint32 values. To get the distance offset the actual match +// distance has to be decreased by 1. A distance offset of 0xffffffff (eos) +// indicates the end of the stream. +func (dc *distCodec) Encode(e *rangeEncoder, dist uint32, l uint32) (err error) { + // Compute the posSlot using nlz32 + var posSlot uint32 + var bits uint32 + if dist < startPosModel { + posSlot = dist + } else { + bits = uint32(30 - nlz32(dist)) + posSlot = startPosModel - 2 + (bits << 1) + posSlot += (dist >> uint(bits)) & 1 + } + + if err = dc.posSlotCodecs[lenState(l)].Encode(e, posSlot); err != nil { + return + } + + switch { + case posSlot < startPosModel: + return nil + case posSlot < endPosModel: + tc := &dc.posModel[posSlot-startPosModel] + return tc.Encode(dist, e) + } + dic := directCodec(bits - alignBits) + if err = dic.Encode(e, dist>>alignBits); err != nil { + return + } + return dc.alignCodec.Encode(dist, e) +} + +// Decode decodes the distance offset using the parameter l. The dist value +// 0xffffffff (eos) indicates the end of the stream. Add one to the distance +// offset to get the actual match distance. +func (dc *distCodec) Decode(d *rangeDecoder, l uint32) (dist uint32, err error) { + posSlot, err := dc.posSlotCodecs[lenState(l)].Decode(d) + if err != nil { + return + } + + // posSlot equals distance + if posSlot < startPosModel { + return posSlot, nil + } + + // posSlot uses the individual models + bits := (posSlot >> 1) - 1 + dist = (2 | (posSlot & 1)) << bits + var u uint32 + if posSlot < endPosModel { + tc := &dc.posModel[posSlot-startPosModel] + if u, err = tc.Decode(d); err != nil { + return 0, err + } + dist += u + return dist, nil + } + + // posSlots use direct encoding and a single model for the four align + // bits. + dic := directCodec(bits - alignBits) + if u, err = dic.Decode(d); err != nil { + return 0, err + } + dist += u << alignBits + if u, err = dc.alignCodec.Decode(d); err != nil { + return 0, err + } + dist += u + return dist, nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoder.go b/vendor/github.com/ulikunitz/xz/lzma/encoder.go new file mode 100644 index 0000000..18ce009 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/encoder.go @@ -0,0 +1,268 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "fmt" + "io" +) + +// opLenMargin provides the upper limit of the number of bytes required +// to encode a single operation. +const opLenMargin = 10 + +// compressFlags control the compression process. +type compressFlags uint32 + +// Values for compressFlags. +const ( + // all data should be compressed, even if compression is not + // optimal. + all compressFlags = 1 << iota +) + +// encoderFlags provide the flags for an encoder. +type encoderFlags uint32 + +// Flags for the encoder. +const ( + // eosMarker requests an EOS marker to be written. + eosMarker encoderFlags = 1 << iota +) + +// Encoder compresses data buffered in the encoder dictionary and writes +// it into a byte writer. +type encoder struct { + dict *encoderDict + state *state + re *rangeEncoder + start int64 + // generate eos marker + marker bool + limit bool + margin int +} + +// newEncoder creates a new encoder. If the byte writer must be +// limited use LimitedByteWriter provided by this package. The flags +// argument supports the eosMarker flag, controlling whether a +// terminating end-of-stream marker must be written. +func newEncoder(bw io.ByteWriter, state *state, dict *encoderDict, + flags encoderFlags) (e *encoder, err error) { + + re, err := newRangeEncoder(bw) + if err != nil { + return nil, err + } + e = &encoder{ + dict: dict, + state: state, + re: re, + marker: flags&eosMarker != 0, + start: dict.Pos(), + margin: opLenMargin, + } + if e.marker { + e.margin += 5 + } + return e, nil +} + +// Write writes the bytes from p into the dictionary. If not enough +// space is available the data in the dictionary buffer will be +// compressed to make additional space available. If the limit of the +// underlying writer has been reached ErrLimit will be returned. +func (e *encoder) Write(p []byte) (n int, err error) { + for { + k, err := e.dict.Write(p[n:]) + n += k + if err == ErrNoSpace { + if err = e.compress(0); err != nil { + return n, err + } + continue + } + return n, err + } +} + +// Reopen reopens the encoder with a new byte writer. +func (e *encoder) Reopen(bw io.ByteWriter) error { + var err error + if e.re, err = newRangeEncoder(bw); err != nil { + return err + } + e.start = e.dict.Pos() + e.limit = false + return nil +} + +// writeLiteral writes a literal into the LZMA stream +func (e *encoder) writeLiteral(l lit) error { + var err error + state, state2, _ := e.state.states(e.dict.Pos()) + if err = e.state.isMatch[state2].Encode(e.re, 0); err != nil { + return err + } + litState := e.state.litState(e.dict.ByteAt(1), e.dict.Pos()) + match := e.dict.ByteAt(int(e.state.rep[0]) + 1) + err = e.state.litCodec.Encode(e.re, l.b, state, match, litState) + if err != nil { + return err + } + e.state.updateStateLiteral() + return nil +} + +// iverson implements the Iverson operator as proposed by Donald Knuth in his +// book Concrete Mathematics. +func iverson(ok bool) uint32 { + if ok { + return 1 + } + return 0 +} + +// writeMatch writes a repetition operation into the operation stream +func (e *encoder) writeMatch(m match) error { + var err error + if !(minDistance <= m.distance && m.distance <= maxDistance) { + panic(fmt.Errorf("match distance %d out of range", m.distance)) + } + dist := uint32(m.distance - minDistance) + if !(minMatchLen <= m.n && m.n <= maxMatchLen) && + !(dist == e.state.rep[0] && m.n == 1) { + panic(fmt.Errorf( + "match length %d out of range; dist %d rep[0] %d", + m.n, dist, e.state.rep[0])) + } + state, state2, posState := e.state.states(e.dict.Pos()) + if err = e.state.isMatch[state2].Encode(e.re, 1); err != nil { + return err + } + g := 0 + for ; g < 4; g++ { + if e.state.rep[g] == dist { + break + } + } + b := iverson(g < 4) + if err = e.state.isRep[state].Encode(e.re, b); err != nil { + return err + } + n := uint32(m.n - minMatchLen) + if b == 0 { + // simple match + e.state.rep[3], e.state.rep[2], e.state.rep[1], e.state.rep[0] = + e.state.rep[2], e.state.rep[1], e.state.rep[0], dist + e.state.updateStateMatch() + if err = e.state.lenCodec.Encode(e.re, n, posState); err != nil { + return err + } + return e.state.distCodec.Encode(e.re, dist, n) + } + b = iverson(g != 0) + if err = e.state.isRepG0[state].Encode(e.re, b); err != nil { + return err + } + if b == 0 { + // g == 0 + b = iverson(m.n != 1) + if err = e.state.isRepG0Long[state2].Encode(e.re, b); err != nil { + return err + } + if b == 0 { + e.state.updateStateShortRep() + return nil + } + } else { + // g in {1,2,3} + b = iverson(g != 1) + if err = e.state.isRepG1[state].Encode(e.re, b); err != nil { + return err + } + if b == 1 { + // g in {2,3} + b = iverson(g != 2) + err = e.state.isRepG2[state].Encode(e.re, b) + if err != nil { + return err + } + if b == 1 { + e.state.rep[3] = e.state.rep[2] + } + e.state.rep[2] = e.state.rep[1] + } + e.state.rep[1] = e.state.rep[0] + e.state.rep[0] = dist + } + e.state.updateStateRep() + return e.state.repLenCodec.Encode(e.re, n, posState) +} + +// writeOp writes a single operation to the range encoder. The function +// checks whether there is enough space available to close the LZMA +// stream. +func (e *encoder) writeOp(op operation) error { + if e.re.Available() < int64(e.margin) { + return ErrLimit + } + switch x := op.(type) { + case lit: + return e.writeLiteral(x) + case match: + return e.writeMatch(x) + default: + panic("unexpected operation") + } +} + +// compress compressed data from the dictionary buffer. If the flag all +// is set, all data in the dictionary buffer will be compressed. The +// function returns ErrLimit if the underlying writer has reached its +// limit. +func (e *encoder) compress(flags compressFlags) error { + n := 0 + if flags&all == 0 { + n = maxMatchLen - 1 + } + d := e.dict + m := d.m + for d.Buffered() > n { + op := m.NextOp(e.state.rep) + if err := e.writeOp(op); err != nil { + return err + } + d.Discard(op.Len()) + } + return nil +} + +// eosMatch is a pseudo operation that indicates the end of the stream. +var eosMatch = match{distance: maxDistance, n: minMatchLen} + +// Close terminates the LZMA stream. If requested the end-of-stream +// marker will be written. If the byte writer limit has been or will be +// reached during compression of the remaining data in the buffer the +// LZMA stream will be closed and data will remain in the buffer. +func (e *encoder) Close() error { + err := e.compress(all) + if err != nil && err != ErrLimit { + return err + } + if e.marker { + if err := e.writeMatch(eosMatch); err != nil { + return err + } + } + err = e.re.Close() + return err +} + +// Compressed returns the number bytes of the input data that been +// compressed. +func (e *encoder) Compressed() int64 { + return e.dict.Pos() - e.start +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go new file mode 100644 index 0000000..9d0fbc7 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go @@ -0,0 +1,149 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "io" +) + +// matcher is an interface that supports the identification of the next +// operation. +type matcher interface { + io.Writer + SetDict(d *encoderDict) + NextOp(rep [4]uint32) operation +} + +// encoderDict provides the dictionary of the encoder. It includes an +// addtional buffer atop of the actual dictionary. +type encoderDict struct { + buf buffer + m matcher + head int64 + capacity int + // preallocated array + data [maxMatchLen]byte +} + +// newEncoderDict creates the encoder dictionary. The argument bufSize +// defines the size of the additional buffer. +func newEncoderDict(dictCap, bufSize int, m matcher) (d *encoderDict, err error) { + if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) { + return nil, errors.New( + "lzma: dictionary capacity out of range") + } + if bufSize < 1 { + return nil, errors.New( + "lzma: buffer size must be larger than zero") + } + d = &encoderDict{ + buf: *newBuffer(dictCap + bufSize), + capacity: dictCap, + m: m, + } + m.SetDict(d) + return d, nil +} + +// Discard discards n bytes. Note that n must not be larger than +// MaxMatchLen. +func (d *encoderDict) Discard(n int) { + p := d.data[:n] + k, _ := d.buf.Read(p) + if k < n { + panic(fmt.Errorf("lzma: can't discard %d bytes", n)) + } + d.head += int64(n) + d.m.Write(p) +} + +// Len returns the data available in the encoder dictionary. +func (d *encoderDict) Len() int { + n := d.buf.Available() + if int64(n) > d.head { + return int(d.head) + } + return n +} + +// DictLen returns the actual length of data in the dictionary. +func (d *encoderDict) DictLen() int { + if d.head < int64(d.capacity) { + return int(d.head) + } + return d.capacity +} + +// Available returns the number of bytes that can be written by a +// following Write call. +func (d *encoderDict) Available() int { + return d.buf.Available() - d.DictLen() +} + +// Write writes data into the dictionary buffer. Note that the position +// of the dictionary head will not be moved. If there is not enough +// space in the buffer ErrNoSpace will be returned. +func (d *encoderDict) Write(p []byte) (n int, err error) { + m := d.Available() + if len(p) > m { + p = p[:m] + err = ErrNoSpace + } + var e error + if n, e = d.buf.Write(p); e != nil { + err = e + } + return n, err +} + +// Pos returns the position of the head. +func (d *encoderDict) Pos() int64 { return d.head } + +// ByteAt returns the byte at the given distance. +func (d *encoderDict) ByteAt(distance int) byte { + if !(0 < distance && distance <= d.Len()) { + return 0 + } + i := d.buf.rear - distance + if i < 0 { + i += len(d.buf.data) + } + return d.buf.data[i] +} + +// CopyN copies the last n bytes from the dictionary into the provided +// writer. This is used for copying uncompressed data into an +// uncompressed segment. +func (d *encoderDict) CopyN(w io.Writer, n int) (written int, err error) { + if n <= 0 { + return 0, nil + } + m := d.Len() + if n > m { + n = m + err = ErrNoSpace + } + i := d.buf.rear - n + var e error + if i < 0 { + i += len(d.buf.data) + if written, e = w.Write(d.buf.data[i:]); e != nil { + return written, e + } + i = 0 + } + var k int + k, e = w.Write(d.buf.data[i:d.buf.rear]) + written += k + if e != nil { + err = e + } + return written, err +} + +// Buffered returns the number of bytes in the buffer. +func (d *encoderDict) Buffered() int { return d.buf.Buffered() } diff --git a/vendor/github.com/ulikunitz/xz/lzma/fox.lzma b/vendor/github.com/ulikunitz/xz/lzma/fox.lzma new file mode 100644 index 0000000..5edad63 Binary files /dev/null and b/vendor/github.com/ulikunitz/xz/lzma/fox.lzma differ diff --git a/vendor/github.com/ulikunitz/xz/lzma/hashtable.go b/vendor/github.com/ulikunitz/xz/lzma/hashtable.go new file mode 100644 index 0000000..d786a97 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/hashtable.go @@ -0,0 +1,309 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + + "github.com/ulikunitz/xz/internal/hash" +) + +/* For compression we need to find byte sequences that match the byte + * sequence at the dictionary head. A hash table is a simple method to + * provide this capability. + */ + +// maxMatches limits the number of matches requested from the Matches +// function. This controls the speed of the overall encoding. +const maxMatches = 16 + +// shortDists defines the number of short distances supported by the +// implementation. +const shortDists = 8 + +// The minimum is somehow arbitrary but the maximum is limited by the +// memory requirements of the hash table. +const ( + minTableExponent = 9 + maxTableExponent = 20 +) + +// newRoller contains the function used to create an instance of the +// hash.Roller. +var newRoller = func(n int) hash.Roller { return hash.NewCyclicPoly(n) } + +// hashTable stores the hash table including the rolling hash method. +// +// We implement chained hashing into a circular buffer. Each entry in +// the circular buffer stores the delta distance to the next position with a +// word that has the same hash value. +type hashTable struct { + dict *encoderDict + // actual hash table + t []int64 + // circular list data with the offset to the next word + data []uint32 + front int + // mask for computing the index for the hash table + mask uint64 + // hash offset; initial value is -int64(wordLen) + hoff int64 + // length of the hashed word + wordLen int + // hash roller for computing the hash values for the Write + // method + wr hash.Roller + // hash roller for computing arbitrary hashes + hr hash.Roller + // preallocated slices + p [maxMatches]int64 + distances [maxMatches + shortDists]int +} + +// hashTableExponent derives the hash table exponent from the dictionary +// capacity. +func hashTableExponent(n uint32) int { + e := 30 - nlz32(n) + switch { + case e < minTableExponent: + e = minTableExponent + case e > maxTableExponent: + e = maxTableExponent + } + return e +} + +// newHashTable creates a new hash table for words of length wordLen +func newHashTable(capacity int, wordLen int) (t *hashTable, err error) { + if !(0 < capacity) { + return nil, errors.New( + "newHashTable: capacity must not be negative") + } + exp := hashTableExponent(uint32(capacity)) + if !(1 <= wordLen && wordLen <= 4) { + return nil, errors.New("newHashTable: " + + "argument wordLen out of range") + } + n := 1 << uint(exp) + if n <= 0 { + panic("newHashTable: exponent is too large") + } + t = &hashTable{ + t: make([]int64, n), + data: make([]uint32, capacity), + mask: (uint64(1) << uint(exp)) - 1, + hoff: -int64(wordLen), + wordLen: wordLen, + wr: newRoller(wordLen), + hr: newRoller(wordLen), + } + return t, nil +} + +func (t *hashTable) SetDict(d *encoderDict) { t.dict = d } + +// buffered returns the number of bytes that are currently hashed. +func (t *hashTable) buffered() int { + n := t.hoff + 1 + switch { + case n <= 0: + return 0 + case n >= int64(len(t.data)): + return len(t.data) + } + return int(n) +} + +// addIndex adds n to an index ensuring that is stays inside the +// circular buffer for the hash chain. +func (t *hashTable) addIndex(i, n int) int { + i += n - len(t.data) + if i < 0 { + i += len(t.data) + } + return i +} + +// putDelta puts the delta instance at the current front of the circular +// chain buffer. +func (t *hashTable) putDelta(delta uint32) { + t.data[t.front] = delta + t.front = t.addIndex(t.front, 1) +} + +// putEntry puts a new entry into the hash table. If there is already a +// value stored it is moved into the circular chain buffer. +func (t *hashTable) putEntry(h uint64, pos int64) { + if pos < 0 { + return + } + i := h & t.mask + old := t.t[i] - 1 + t.t[i] = pos + 1 + var delta int64 + if old >= 0 { + delta = pos - old + if delta > 1<<32-1 || delta > int64(t.buffered()) { + delta = 0 + } + } + t.putDelta(uint32(delta)) +} + +// WriteByte converts a single byte into a hash and puts them into the hash +// table. +func (t *hashTable) WriteByte(b byte) error { + h := t.wr.RollByte(b) + t.hoff++ + t.putEntry(h, t.hoff) + return nil +} + +// Write converts the bytes provided into hash tables and stores the +// abbreviated offsets into the hash table. The method will never return an +// error. +func (t *hashTable) Write(p []byte) (n int, err error) { + for _, b := range p { + // WriteByte doesn't generate an error. + t.WriteByte(b) + } + return len(p), nil +} + +// getMatches the matches for a specific hash. The functions returns the +// number of positions found. +// +// TODO: Make a getDistances because that we are actually interested in. +func (t *hashTable) getMatches(h uint64, positions []int64) (n int) { + if t.hoff < 0 || len(positions) == 0 { + return 0 + } + buffered := t.buffered() + tailPos := t.hoff + 1 - int64(buffered) + rear := t.front - buffered + if rear >= 0 { + rear -= len(t.data) + } + // get the slot for the hash + pos := t.t[h&t.mask] - 1 + delta := pos - tailPos + for { + if delta < 0 { + return n + } + positions[n] = tailPos + delta + n++ + if n >= len(positions) { + return n + } + i := rear + int(delta) + if i < 0 { + i += len(t.data) + } + u := t.data[i] + if u == 0 { + return n + } + delta -= int64(u) + } +} + +// hash computes the rolling hash for the word stored in p. For correct +// results its length must be equal to t.wordLen. +func (t *hashTable) hash(p []byte) uint64 { + var h uint64 + for _, b := range p { + h = t.hr.RollByte(b) + } + return h +} + +// Matches fills the positions slice with potential matches. The +// functions returns the number of positions filled into positions. The +// byte slice p must have word length of the hash table. +func (t *hashTable) Matches(p []byte, positions []int64) int { + if len(p) != t.wordLen { + panic(fmt.Errorf( + "byte slice must have length %d", t.wordLen)) + } + h := t.hash(p) + return t.getMatches(h, positions) +} + +// NextOp identifies the next operation using the hash table. +// +// TODO: Use all repetitions to find matches. +func (t *hashTable) NextOp(rep [4]uint32) operation { + // get positions + data := t.dict.data[:maxMatchLen] + n, _ := t.dict.buf.Peek(data) + data = data[:n] + var p []int64 + if n < t.wordLen { + p = t.p[:0] + } else { + p = t.p[:maxMatches] + n = t.Matches(data[:t.wordLen], p) + p = p[:n] + } + + // convert positions in potential distances + head := t.dict.head + dists := append(t.distances[:0], 1, 2, 3, 4, 5, 6, 7, 8) + for _, pos := range p { + dis := int(head - pos) + if dis > shortDists { + dists = append(dists, dis) + } + } + + // check distances + var m match + dictLen := t.dict.DictLen() + for _, dist := range dists { + if dist > dictLen { + continue + } + + // Here comes a trick. We are only interested in matches + // that are longer than the matches we have been found + // before. So before we test the whole byte sequence at + // the given distance, we test the first byte that would + // make the match longer. If it doesn't match the byte + // to match, we don't to care any longer. + i := t.dict.buf.rear - dist + m.n + if i < 0 { + i += len(t.dict.buf.data) + } + if t.dict.buf.data[i] != data[m.n] { + // We can't get a longer match. Jump to the next + // distance. + continue + } + + n := t.dict.buf.matchLen(dist, data) + switch n { + case 0: + continue + case 1: + if uint32(dist-minDistance) != rep[0] { + continue + } + } + if n > m.n { + m = match{int64(dist), n} + if n == len(data) { + // No better match will be found. + break + } + } + } + + if m.n == 0 { + return lit{data[0]} + } + return m +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/header.go b/vendor/github.com/ulikunitz/xz/lzma/header.go new file mode 100644 index 0000000..bc70896 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/header.go @@ -0,0 +1,167 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" +) + +// uint32LE reads an uint32 integer from a byte slice +func uint32LE(b []byte) uint32 { + x := uint32(b[3]) << 24 + x |= uint32(b[2]) << 16 + x |= uint32(b[1]) << 8 + x |= uint32(b[0]) + return x +} + +// uint64LE converts the uint64 value stored as little endian to an uint64 +// value. +func uint64LE(b []byte) uint64 { + x := uint64(b[7]) << 56 + x |= uint64(b[6]) << 48 + x |= uint64(b[5]) << 40 + x |= uint64(b[4]) << 32 + x |= uint64(b[3]) << 24 + x |= uint64(b[2]) << 16 + x |= uint64(b[1]) << 8 + x |= uint64(b[0]) + return x +} + +// putUint32LE puts an uint32 integer into a byte slice that must have at least +// a length of 4 bytes. +func putUint32LE(b []byte, x uint32) { + b[0] = byte(x) + b[1] = byte(x >> 8) + b[2] = byte(x >> 16) + b[3] = byte(x >> 24) +} + +// putUint64LE puts the uint64 value into the byte slice as little endian +// value. The byte slice b must have at least place for 8 bytes. +func putUint64LE(b []byte, x uint64) { + b[0] = byte(x) + b[1] = byte(x >> 8) + b[2] = byte(x >> 16) + b[3] = byte(x >> 24) + b[4] = byte(x >> 32) + b[5] = byte(x >> 40) + b[6] = byte(x >> 48) + b[7] = byte(x >> 56) +} + +// noHeaderSize defines the value of the length field in the LZMA header. +const noHeaderSize uint64 = 1<<64 - 1 + +// HeaderLen provides the length of the LZMA file header. +const HeaderLen = 13 + +// header represents the header of an LZMA file. +type header struct { + properties Properties + dictCap int + // uncompressed size; negative value if no size is given + size int64 +} + +// marshalBinary marshals the header. +func (h *header) marshalBinary() (data []byte, err error) { + if err = h.properties.verify(); err != nil { + return nil, err + } + if !(0 <= h.dictCap && int64(h.dictCap) <= MaxDictCap) { + return nil, fmt.Errorf("lzma: DictCap %d out of range", + h.dictCap) + } + + data = make([]byte, 13) + + // property byte + data[0] = h.properties.Code() + + // dictionary capacity + putUint32LE(data[1:5], uint32(h.dictCap)) + + // uncompressed size + var s uint64 + if h.size > 0 { + s = uint64(h.size) + } else { + s = noHeaderSize + } + putUint64LE(data[5:], s) + + return data, nil +} + +// unmarshalBinary unmarshals the header. +func (h *header) unmarshalBinary(data []byte) error { + if len(data) != HeaderLen { + return errors.New("lzma.unmarshalBinary: data has wrong length") + } + + // properties + var err error + if h.properties, err = PropertiesForCode(data[0]); err != nil { + return err + } + + // dictionary capacity + h.dictCap = int(uint32LE(data[1:])) + if h.dictCap < 0 { + return errors.New( + "LZMA header: dictionary capacity exceeds maximum " + + "integer") + } + + // uncompressed size + s := uint64LE(data[5:]) + if s == noHeaderSize { + h.size = -1 + } else { + h.size = int64(s) + if h.size < 0 { + return errors.New( + "LZMA header: uncompressed size " + + "out of int64 range") + } + } + + return nil +} + +// validDictCap checks whether the dictionary capacity is correct. This +// is used to weed out wrong file headers. +func validDictCap(dictcap int) bool { + if int64(dictcap) == MaxDictCap { + return true + } + for n := uint(10); n < 32; n++ { + if dictcap == 1<= 10 or 2^32-1. If +// there is an explicit size it must not exceed 256 GiB. The length of +// the data argument must be HeaderLen. +func ValidHeader(data []byte) bool { + var h header + if err := h.unmarshalBinary(data); err != nil { + return false + } + if !validDictCap(h.dictCap) { + return false + } + return h.size < 0 || h.size <= 1<<38 +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/header2.go b/vendor/github.com/ulikunitz/xz/lzma/header2.go new file mode 100644 index 0000000..ac6a71a --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/header2.go @@ -0,0 +1,398 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "io" +) + +const ( + // maximum size of compressed data in a chunk + maxCompressed = 1 << 16 + // maximum size of uncompressed data in a chunk + maxUncompressed = 1 << 21 +) + +// chunkType represents the type of an LZMA2 chunk. Note that this +// value is an internal representation and no actual encoding of a LZMA2 +// chunk header. +type chunkType byte + +// Possible values for the chunk type. +const ( + // end of stream + cEOS chunkType = iota + // uncompressed; reset dictionary + cUD + // uncompressed; no reset of dictionary + cU + // LZMA compressed; no reset + cL + // LZMA compressed; reset state + cLR + // LZMA compressed; reset state; new property value + cLRN + // LZMA compressed; reset state; new property value; reset dictionary + cLRND +) + +// chunkTypeStrings provide a string representation for the chunk types. +var chunkTypeStrings = [...]string{ + cEOS: "EOS", + cU: "U", + cUD: "UD", + cL: "L", + cLR: "LR", + cLRN: "LRN", + cLRND: "LRND", +} + +// String returns a string representation of the chunk type. +func (c chunkType) String() string { + if !(cEOS <= c && c <= cLRND) { + return "unknown" + } + return chunkTypeStrings[c] +} + +// Actual encodings for the chunk types in the value. Note that the high +// uncompressed size bits are stored in the header byte additionally. +const ( + hEOS = 0 + hUD = 1 + hU = 2 + hL = 1 << 7 + hLR = 1<<7 | 1<<5 + hLRN = 1<<7 | 1<<6 + hLRND = 1<<7 | 1<<6 | 1<<5 +) + +// errHeaderByte indicates an unsupported value for the chunk header +// byte. These bytes starts the variable-length chunk header. +var errHeaderByte = errors.New("lzma: unsupported chunk header byte") + +// headerChunkType converts the header byte into a chunk type. It +// ignores the uncompressed size bits in the chunk header byte. +func headerChunkType(h byte) (c chunkType, err error) { + if h&hL == 0 { + // no compression + switch h { + case hEOS: + c = cEOS + case hUD: + c = cUD + case hU: + c = cU + default: + return 0, errHeaderByte + } + return + } + switch h & hLRND { + case hL: + c = cL + case hLR: + c = cLR + case hLRN: + c = cLRN + case hLRND: + c = cLRND + default: + return 0, errHeaderByte + } + return +} + +// uncompressedHeaderLen provides the length of an uncompressed header +const uncompressedHeaderLen = 3 + +// headerLen returns the length of the LZMA2 header for a given chunk +// type. +func headerLen(c chunkType) int { + switch c { + case cEOS: + return 1 + case cU, cUD: + return uncompressedHeaderLen + case cL, cLR: + return 5 + case cLRN, cLRND: + return 6 + } + panic(fmt.Errorf("unsupported chunk type %d", c)) +} + +// chunkHeader represents the contents of a chunk header. +type chunkHeader struct { + ctype chunkType + uncompressed uint32 + compressed uint16 + props Properties +} + +// String returns a string representation of the chunk header. +func (h *chunkHeader) String() string { + return fmt.Sprintf("%s %d %d %s", h.ctype, h.uncompressed, + h.compressed, &h.props) +} + +// UnmarshalBinary reads the content of the chunk header from the data +// slice. The slice must have the correct length. +func (h *chunkHeader) UnmarshalBinary(data []byte) error { + if len(data) == 0 { + return errors.New("no data") + } + c, err := headerChunkType(data[0]) + if err != nil { + return err + } + + n := headerLen(c) + if len(data) < n { + return errors.New("incomplete data") + } + if len(data) > n { + return errors.New("invalid data length") + } + + *h = chunkHeader{ctype: c} + if c == cEOS { + return nil + } + + h.uncompressed = uint32(uint16BE(data[1:3])) + if c <= cU { + return nil + } + h.uncompressed |= uint32(data[0]&^hLRND) << 16 + + h.compressed = uint16BE(data[3:5]) + if c <= cLR { + return nil + } + + h.props, err = PropertiesForCode(data[5]) + return err +} + +// MarshalBinary encodes the chunk header value. The function checks +// whether the content of the chunk header is correct. +func (h *chunkHeader) MarshalBinary() (data []byte, err error) { + if h.ctype > cLRND { + return nil, errors.New("invalid chunk type") + } + if err = h.props.verify(); err != nil { + return nil, err + } + + data = make([]byte, headerLen(h.ctype)) + + switch h.ctype { + case cEOS: + return data, nil + case cUD: + data[0] = hUD + case cU: + data[0] = hU + case cL: + data[0] = hL + case cLR: + data[0] = hLR + case cLRN: + data[0] = hLRN + case cLRND: + data[0] = hLRND + } + + putUint16BE(data[1:3], uint16(h.uncompressed)) + if h.ctype <= cU { + return data, nil + } + data[0] |= byte(h.uncompressed>>16) &^ hLRND + + putUint16BE(data[3:5], h.compressed) + if h.ctype <= cLR { + return data, nil + } + + data[5] = h.props.Code() + return data, nil +} + +// readChunkHeader reads the chunk header from the IO reader. +func readChunkHeader(r io.Reader) (h *chunkHeader, err error) { + p := make([]byte, 1, 6) + if _, err = io.ReadFull(r, p); err != nil { + return + } + c, err := headerChunkType(p[0]) + if err != nil { + return + } + p = p[:headerLen(c)] + if _, err = io.ReadFull(r, p[1:]); err != nil { + return + } + h = new(chunkHeader) + if err = h.UnmarshalBinary(p); err != nil { + return nil, err + } + return h, nil +} + +// uint16BE converts a big-endian uint16 representation to an uint16 +// value. +func uint16BE(p []byte) uint16 { + return uint16(p[0])<<8 | uint16(p[1]) +} + +// putUint16BE puts the big-endian uint16 presentation into the given +// slice. +func putUint16BE(p []byte, x uint16) { + p[0] = byte(x >> 8) + p[1] = byte(x) +} + +// chunkState is used to manage the state of the chunks +type chunkState byte + +// start and stop define the initial and terminating state of the chunk +// state +const ( + start chunkState = 'S' + stop = 'T' +) + +// errors for the chunk state handling +var ( + errChunkType = errors.New("lzma: unexpected chunk type") + errState = errors.New("lzma: wrong chunk state") +) + +// next transitions state based on chunk type input +func (c *chunkState) next(ctype chunkType) error { + switch *c { + // start state + case 'S': + switch ctype { + case cEOS: + *c = 'T' + case cUD: + *c = 'R' + case cLRND: + *c = 'L' + default: + return errChunkType + } + // normal LZMA mode + case 'L': + switch ctype { + case cEOS: + *c = 'T' + case cUD: + *c = 'R' + case cU: + *c = 'U' + case cL, cLR, cLRN, cLRND: + break + default: + return errChunkType + } + // reset required + case 'R': + switch ctype { + case cEOS: + *c = 'T' + case cUD, cU: + break + case cLRN, cLRND: + *c = 'L' + default: + return errChunkType + } + // uncompressed + case 'U': + switch ctype { + case cEOS: + *c = 'T' + case cUD: + *c = 'R' + case cU: + break + case cL, cLR, cLRN, cLRND: + *c = 'L' + default: + return errChunkType + } + // terminal state + case 'T': + return errChunkType + default: + return errState + } + return nil +} + +// defaultChunkType returns the default chunk type for each chunk state. +func (c chunkState) defaultChunkType() chunkType { + switch c { + case 'S': + return cLRND + case 'L', 'U': + return cL + case 'R': + return cLRN + default: + // no error + return cEOS + } +} + +// maxDictCap defines the maximum dictionary capacity supported by the +// LZMA2 dictionary capacity encoding. +const maxDictCap = 1<<32 - 1 + +// maxDictCapCode defines the maximum dictionary capacity code. +const maxDictCapCode = 40 + +// The function decodes the dictionary capacity byte, but doesn't change +// for the correct range of the given byte. +func decodeDictCap(c byte) int64 { + return (2 | int64(c)&1) << (11 + (c>>1)&0x1f) +} + +// DecodeDictCap decodes the encoded dictionary capacity. The function +// returns an error if the code is out of range. +func DecodeDictCap(c byte) (n int64, err error) { + if c >= maxDictCapCode { + if c == maxDictCapCode { + return maxDictCap, nil + } + return 0, errors.New("lzma: invalid dictionary size code") + } + return decodeDictCap(c), nil +} + +// EncodeDictCap encodes a dictionary capacity. The function returns the +// code for the capacity that is greater or equal n. If n exceeds the +// maximum support dictionary capacity, the maximum value is returned. +func EncodeDictCap(n int64) byte { + a, b := byte(0), byte(40) + for a < b { + c := a + (b-a)>>1 + m := decodeDictCap(c) + if n <= m { + if n == m { + return c + } + b = c + } else { + a = c + 1 + } + } + return a +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go new file mode 100644 index 0000000..e517730 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go @@ -0,0 +1,129 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import "errors" + +// maxPosBits defines the number of bits of the position value that are used to +// to compute the posState value. The value is used to select the tree codec +// for length encoding and decoding. +const maxPosBits = 4 + +// minMatchLen and maxMatchLen give the minimum and maximum values for +// encoding and decoding length values. minMatchLen is also used as base +// for the encoded length values. +const ( + minMatchLen = 2 + maxMatchLen = minMatchLen + 16 + 256 - 1 +) + +// lengthCodec support the encoding of the length value. +type lengthCodec struct { + choice [2]prob + low [1 << maxPosBits]treeCodec + mid [1 << maxPosBits]treeCodec + high treeCodec +} + +// deepcopy initializes the lc value as deep copy of the source value. +func (lc *lengthCodec) deepcopy(src *lengthCodec) { + if lc == src { + return + } + lc.choice = src.choice + for i := range lc.low { + lc.low[i].deepcopy(&src.low[i]) + } + for i := range lc.mid { + lc.mid[i].deepcopy(&src.mid[i]) + } + lc.high.deepcopy(&src.high) +} + +// init initializes a new length codec. +func (lc *lengthCodec) init() { + for i := range lc.choice { + lc.choice[i] = probInit + } + for i := range lc.low { + lc.low[i] = makeTreeCodec(3) + } + for i := range lc.mid { + lc.mid[i] = makeTreeCodec(3) + } + lc.high = makeTreeCodec(8) +} + +// lBits gives the number of bits used for the encoding of the l value +// provided to the range encoder. +func lBits(l uint32) int { + switch { + case l < 8: + return 4 + case l < 16: + return 5 + default: + return 10 + } +} + +// Encode encodes the length offset. The length offset l can be compute by +// subtracting minMatchLen (2) from the actual length. +// +// l = length - minMatchLen +// +func (lc *lengthCodec) Encode(e *rangeEncoder, l uint32, posState uint32, +) (err error) { + if l > maxMatchLen-minMatchLen { + return errors.New("lengthCodec.Encode: l out of range") + } + if l < 8 { + if err = lc.choice[0].Encode(e, 0); err != nil { + return + } + return lc.low[posState].Encode(e, l) + } + if err = lc.choice[0].Encode(e, 1); err != nil { + return + } + if l < 16 { + if err = lc.choice[1].Encode(e, 0); err != nil { + return + } + return lc.mid[posState].Encode(e, l-8) + } + if err = lc.choice[1].Encode(e, 1); err != nil { + return + } + if err = lc.high.Encode(e, l-16); err != nil { + return + } + return nil +} + +// Decode reads the length offset. Add minMatchLen to compute the actual length +// to the length offset l. +func (lc *lengthCodec) Decode(d *rangeDecoder, posState uint32, +) (l uint32, err error) { + var b uint32 + if b, err = lc.choice[0].Decode(d); err != nil { + return + } + if b == 0 { + l, err = lc.low[posState].Decode(d) + return + } + if b, err = lc.choice[1].Decode(d); err != nil { + return + } + if b == 0 { + l, err = lc.mid[posState].Decode(d) + l += 8 + return + } + l, err = lc.high.Decode(d) + l += 16 + return +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go new file mode 100644 index 0000000..c949d6e --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go @@ -0,0 +1,132 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// literalCodec supports the encoding of literal. It provides 768 probability +// values per literal state. The upper 512 probabilities are used with the +// context of a match bit. +type literalCodec struct { + probs []prob +} + +// deepcopy initializes literal codec c as a deep copy of the source. +func (c *literalCodec) deepcopy(src *literalCodec) { + if c == src { + return + } + c.probs = make([]prob, len(src.probs)) + copy(c.probs, src.probs) +} + +// init initializes the literal codec. +func (c *literalCodec) init(lc, lp int) { + switch { + case !(minLC <= lc && lc <= maxLC): + panic("lc out of range") + case !(minLP <= lp && lp <= maxLP): + panic("lp out of range") + } + c.probs = make([]prob, 0x300<= 7 { + m := uint32(match) + for { + matchBit := (m >> 7) & 1 + m <<= 1 + bit := (r >> 7) & 1 + r <<= 1 + i := ((1 + matchBit) << 8) | symbol + if err = probs[i].Encode(e, bit); err != nil { + return + } + symbol = (symbol << 1) | bit + if matchBit != bit { + break + } + if symbol >= 0x100 { + break + } + } + } + for symbol < 0x100 { + bit := (r >> 7) & 1 + r <<= 1 + if err = probs[symbol].Encode(e, bit); err != nil { + return + } + symbol = (symbol << 1) | bit + } + return nil +} + +// Decode decodes a literal byte using the range decoder as well as the LZMA +// state, a match byte, and the literal state. +func (c *literalCodec) Decode(d *rangeDecoder, + state uint32, match byte, litState uint32, +) (s byte, err error) { + k := litState * 0x300 + probs := c.probs[k : k+0x300] + symbol := uint32(1) + if state >= 7 { + m := uint32(match) + for { + matchBit := (m >> 7) & 1 + m <<= 1 + i := ((1 + matchBit) << 8) | symbol + bit, err := d.DecodeBit(&probs[i]) + if err != nil { + return 0, err + } + symbol = (symbol << 1) | bit + if matchBit != bit { + break + } + if symbol >= 0x100 { + break + } + } + } + for symbol < 0x100 { + bit, err := d.DecodeBit(&probs[symbol]) + if err != nil { + return 0, err + } + symbol = (symbol << 1) | bit + } + s = byte(symbol - 0x100) + return s, nil +} + +// minLC and maxLC define the range for LC values. +const ( + minLC = 0 + maxLC = 8 +) + +// minLC and maxLC define the range for LP values. +const ( + minLP = 0 + maxLP = 4 +) + +// minState and maxState define a range for the state values stored in +// the State values. +const ( + minState = 0 + maxState = 11 +) diff --git a/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go b/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go new file mode 100644 index 0000000..4a244eb --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go @@ -0,0 +1,52 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import "errors" + +// MatchAlgorithm identifies an algorithm to find matches in the +// dictionary. +type MatchAlgorithm byte + +// Supported matcher algorithms. +const ( + HashTable4 MatchAlgorithm = iota + BinaryTree +) + +// maStrings are used by the String method. +var maStrings = map[MatchAlgorithm]string{ + HashTable4: "HashTable4", + BinaryTree: "BinaryTree", +} + +// String returns a string representation of the Matcher. +func (a MatchAlgorithm) String() string { + if s, ok := maStrings[a]; ok { + return s + } + return "unknown" +} + +var errUnsupportedMatchAlgorithm = errors.New( + "lzma: unsupported match algorithm value") + +// verify checks whether the matcher value is supported. +func (a MatchAlgorithm) verify() error { + if _, ok := maStrings[a]; !ok { + return errUnsupportedMatchAlgorithm + } + return nil +} + +func (a MatchAlgorithm) new(dictCap int) (m matcher, err error) { + switch a { + case HashTable4: + return newHashTable(dictCap, 4) + case BinaryTree: + return newBinTree(dictCap) + } + return nil, errUnsupportedMatchAlgorithm +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/operation.go b/vendor/github.com/ulikunitz/xz/lzma/operation.go new file mode 100644 index 0000000..733bb99 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/operation.go @@ -0,0 +1,80 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "unicode" +) + +// operation represents an operation on the dictionary during encoding or +// decoding. +type operation interface { + Len() int +} + +// rep represents a repetition at the given distance and the given length +type match struct { + // supports all possible distance values, including the eos marker + distance int64 + // length + n int +} + +// verify checks whether the match is valid. If that is not the case an +// error is returned. +func (m match) verify() error { + if !(minDistance <= m.distance && m.distance <= maxDistance) { + return errors.New("distance out of range") + } + if !(1 <= m.n && m.n <= maxMatchLen) { + return errors.New("length out of range") + } + return nil +} + +// l return the l-value for the match, which is the difference of length +// n and 2. +func (m match) l() uint32 { + return uint32(m.n - minMatchLen) +} + +// dist returns the dist value for the match, which is one less of the +// distance stored in the match. +func (m match) dist() uint32 { + return uint32(m.distance - minDistance) +} + +// Len returns the number of bytes matched. +func (m match) Len() int { + return m.n +} + +// String returns a string representation for the repetition. +func (m match) String() string { + return fmt.Sprintf("M{%d,%d}", m.distance, m.n) +} + +// lit represents a single byte literal. +type lit struct { + b byte +} + +// Len returns 1 for the single byte literal. +func (l lit) Len() int { + return 1 +} + +// String returns a string representation for the literal. +func (l lit) String() string { + var c byte + if unicode.IsPrint(rune(l.b)) { + c = l.b + } else { + c = '.' + } + return fmt.Sprintf("L{%c/%02x}", c, l.b) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/prob.go b/vendor/github.com/ulikunitz/xz/lzma/prob.go new file mode 100644 index 0000000..24d50ec --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/prob.go @@ -0,0 +1,53 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// movebits defines the number of bits used for the updates of probability +// values. +const movebits = 5 + +// probbits defines the number of bits of a probability value. +const probbits = 11 + +// probInit defines 0.5 as initial value for prob values. +const probInit prob = 1 << (probbits - 1) + +// Type prob represents probabilities. The type can also be used to encode and +// decode single bits. +type prob uint16 + +// Dec decreases the probability. The decrease is proportional to the +// probability value. +func (p *prob) dec() { + *p -= *p >> movebits +} + +// Inc increases the probability. The Increase is proportional to the +// difference of 1 and the probability value. +func (p *prob) inc() { + *p += ((1 << probbits) - *p) >> movebits +} + +// Computes the new bound for a given range using the probability value. +func (p prob) bound(r uint32) uint32 { + return (r >> probbits) * uint32(p) +} + +// Bits returns 1. One is the number of bits that can be encoded or decoded +// with a single prob value. +func (p prob) Bits() int { + return 1 +} + +// Encode encodes the least-significant bit of v. Note that the p value will be +// changed. +func (p *prob) Encode(e *rangeEncoder, v uint32) error { + return e.EncodeBit(v, p) +} + +// Decode decodes a single bit. Note that the p value will change. +func (p *prob) Decode(d *rangeDecoder) (v uint32, err error) { + return d.DecodeBit(p) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/properties.go b/vendor/github.com/ulikunitz/xz/lzma/properties.go new file mode 100644 index 0000000..23418e2 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/properties.go @@ -0,0 +1,69 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" +) + +// maximum and minimum values for the LZMA properties. +const ( + minPB = 0 + maxPB = 4 +) + +// maxPropertyCode is the possible maximum of a properties code byte. +const maxPropertyCode = (maxPB+1)*(maxLP+1)*(maxLC+1) - 1 + +// Properties contains the parameters LC, LP and PB. The parameter LC +// defines the number of literal context bits; parameter LP the number +// of literal position bits and PB the number of position bits. +type Properties struct { + LC int + LP int + PB int +} + +// String returns the properties in a string representation. +func (p *Properties) String() string { + return fmt.Sprintf("LC %d LP %d PB %d", p.LC, p.LP, p.PB) +} + +// PropertiesForCode converts a properties code byte into a Properties value. +func PropertiesForCode(code byte) (p Properties, err error) { + if code > maxPropertyCode { + return p, errors.New("lzma: invalid properties code") + } + p.LC = int(code % 9) + code /= 9 + p.LP = int(code % 5) + code /= 5 + p.PB = int(code % 5) + return p, err +} + +// verify checks the properties for correctness. +func (p *Properties) verify() error { + if p == nil { + return errors.New("lzma: properties are nil") + } + if !(minLC <= p.LC && p.LC <= maxLC) { + return errors.New("lzma: lc out of range") + } + if !(minLP <= p.LP && p.LP <= maxLP) { + return errors.New("lzma: lp out of range") + } + if !(minPB <= p.PB && p.PB <= maxPB) { + return errors.New("lzma: pb out of range") + } + return nil +} + +// Code converts the properties to a byte. The function assumes that +// the properties components are all in range. +func (p Properties) Code() byte { + return byte((p.PB*5+p.LP)*9 + p.LC) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go new file mode 100644 index 0000000..6361c5e --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go @@ -0,0 +1,248 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" +) + +// rangeEncoder implements range encoding of single bits. The low value can +// overflow therefore we need uint64. The cache value is used to handle +// overflows. +type rangeEncoder struct { + lbw *LimitedByteWriter + nrange uint32 + low uint64 + cacheLen int64 + cache byte +} + +// maxInt64 provides the maximal value of the int64 type +const maxInt64 = 1<<63 - 1 + +// newRangeEncoder creates a new range encoder. +func newRangeEncoder(bw io.ByteWriter) (re *rangeEncoder, err error) { + lbw, ok := bw.(*LimitedByteWriter) + if !ok { + lbw = &LimitedByteWriter{BW: bw, N: maxInt64} + } + return &rangeEncoder{ + lbw: lbw, + nrange: 0xffffffff, + cacheLen: 1}, nil +} + +// Available returns the number of bytes that still can be written. The +// method takes the bytes that will be currently written by Close into +// account. +func (e *rangeEncoder) Available() int64 { + return e.lbw.N - (e.cacheLen + 4) +} + +// writeByte writes a single byte to the underlying writer. An error is +// returned if the limit is reached. The written byte will be counted if +// the underlying writer doesn't return an error. +func (e *rangeEncoder) writeByte(c byte) error { + if e.Available() < 1 { + return ErrLimit + } + return e.lbw.WriteByte(c) +} + +// DirectEncodeBit encodes the least-significant bit of b with probability 1/2. +func (e *rangeEncoder) DirectEncodeBit(b uint32) error { + e.nrange >>= 1 + e.low += uint64(e.nrange) & (0 - (uint64(b) & 1)) + + // normalize + const top = 1 << 24 + if e.nrange >= top { + return nil + } + e.nrange <<= 8 + return e.shiftLow() +} + +// EncodeBit encodes the least significant bit of b. The p value will be +// updated by the function depending on the bit encoded. +func (e *rangeEncoder) EncodeBit(b uint32, p *prob) error { + bound := p.bound(e.nrange) + if b&1 == 0 { + e.nrange = bound + p.inc() + } else { + e.low += uint64(bound) + e.nrange -= bound + p.dec() + } + + // normalize + const top = 1 << 24 + if e.nrange >= top { + return nil + } + e.nrange <<= 8 + return e.shiftLow() +} + +// Close writes a complete copy of the low value. +func (e *rangeEncoder) Close() error { + for i := 0; i < 5; i++ { + if err := e.shiftLow(); err != nil { + return err + } + } + return nil +} + +// shiftLow shifts the low value for 8 bit. The shifted byte is written into +// the byte writer. The cache value is used to handle overflows. +func (e *rangeEncoder) shiftLow() error { + if uint32(e.low) < 0xff000000 || (e.low>>32) != 0 { + tmp := e.cache + for { + err := e.writeByte(tmp + byte(e.low>>32)) + if err != nil { + return err + } + tmp = 0xff + e.cacheLen-- + if e.cacheLen <= 0 { + if e.cacheLen < 0 { + panic("negative cacheLen") + } + break + } + } + e.cache = byte(uint32(e.low) >> 24) + } + e.cacheLen++ + e.low = uint64(uint32(e.low) << 8) + return nil +} + +// rangeDecoder decodes single bits of the range encoding stream. +type rangeDecoder struct { + br io.ByteReader + nrange uint32 + code uint32 +} + +// init initializes the range decoder, by reading from the byte reader. +func (d *rangeDecoder) init() error { + d.nrange = 0xffffffff + d.code = 0 + + b, err := d.br.ReadByte() + if err != nil { + return err + } + if b != 0 { + return errors.New("newRangeDecoder: first byte not zero") + } + + for i := 0; i < 4; i++ { + if err = d.updateCode(); err != nil { + return err + } + } + + if d.code >= d.nrange { + return errors.New("newRangeDecoder: d.code >= d.nrange") + } + + return nil +} + +// newRangeDecoder initializes a range decoder. It reads five bytes from the +// reader and therefore may return an error. +func newRangeDecoder(br io.ByteReader) (d *rangeDecoder, err error) { + d = &rangeDecoder{br: br, nrange: 0xffffffff} + + b, err := d.br.ReadByte() + if err != nil { + return nil, err + } + if b != 0 { + return nil, errors.New("newRangeDecoder: first byte not zero") + } + + for i := 0; i < 4; i++ { + if err = d.updateCode(); err != nil { + return nil, err + } + } + + if d.code >= d.nrange { + return nil, errors.New("newRangeDecoder: d.code >= d.nrange") + } + + return d, nil +} + +// possiblyAtEnd checks whether the decoder may be at the end of the stream. +func (d *rangeDecoder) possiblyAtEnd() bool { + return d.code == 0 +} + +// DirectDecodeBit decodes a bit with probability 1/2. The return value b will +// contain the bit at the least-significant position. All other bits will be +// zero. +func (d *rangeDecoder) DirectDecodeBit() (b uint32, err error) { + d.nrange >>= 1 + d.code -= d.nrange + t := 0 - (d.code >> 31) + d.code += d.nrange & t + b = (t + 1) & 1 + + // d.code will stay less then d.nrange + + // normalize + // assume d.code < d.nrange + const top = 1 << 24 + if d.nrange >= top { + return b, nil + } + d.nrange <<= 8 + // d.code < d.nrange will be maintained + return b, d.updateCode() +} + +// decodeBit decodes a single bit. The bit will be returned at the +// least-significant position. All other bits will be zero. The probability +// value will be updated. +func (d *rangeDecoder) DecodeBit(p *prob) (b uint32, err error) { + bound := p.bound(d.nrange) + if d.code < bound { + d.nrange = bound + p.inc() + b = 0 + } else { + d.code -= bound + d.nrange -= bound + p.dec() + b = 1 + } + // normalize + // assume d.code < d.nrange + const top = 1 << 24 + if d.nrange >= top { + return b, nil + } + d.nrange <<= 8 + // d.code < d.nrange will be maintained + return b, d.updateCode() +} + +// updateCode reads a new byte into the code. +func (d *rangeDecoder) updateCode() error { + b, err := d.br.ReadByte() + if err != nil { + return err + } + d.code = (d.code << 8) | uint32(b) + return nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader.go b/vendor/github.com/ulikunitz/xz/lzma/reader.go new file mode 100644 index 0000000..2ef3dca --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/reader.go @@ -0,0 +1,100 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lzma supports the decoding and encoding of LZMA streams. +// Reader and Writer support the classic LZMA format. Reader2 and +// Writer2 support the decoding and encoding of LZMA2 streams. +// +// The package is written completely in Go and doesn't rely on any external +// library. +package lzma + +import ( + "errors" + "io" +) + +// ReaderConfig stores the parameters for the reader of the classic LZMA +// format. +type ReaderConfig struct { + DictCap int +} + +// fill converts the zero values of the configuration to the default values. +func (c *ReaderConfig) fill() { + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } +} + +// Verify checks the reader configuration for errors. Zero values will +// be replaced by default values. +func (c *ReaderConfig) Verify() error { + c.fill() + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + return nil +} + +// Reader provides a reader for LZMA files or streams. +type Reader struct { + lzma io.Reader + h header + d *decoder +} + +// NewReader creates a new reader for an LZMA stream using the classic +// format. NewReader reads and checks the header of the LZMA stream. +func NewReader(lzma io.Reader) (r *Reader, err error) { + return ReaderConfig{}.NewReader(lzma) +} + +// NewReader creates a new reader for an LZMA stream in the classic +// format. The function reads and verifies the the header of the LZMA +// stream. +func (c ReaderConfig) NewReader(lzma io.Reader) (r *Reader, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + data := make([]byte, HeaderLen) + if _, err := io.ReadFull(lzma, data); err != nil { + if err == io.EOF { + return nil, errors.New("lzma: unexpected EOF") + } + return nil, err + } + r = &Reader{lzma: lzma} + if err = r.h.unmarshalBinary(data); err != nil { + return nil, err + } + if r.h.dictCap < MinDictCap { + return nil, errors.New("lzma: dictionary capacity too small") + } + dictCap := r.h.dictCap + if c.DictCap > dictCap { + dictCap = c.DictCap + } + + state := newState(r.h.properties) + dict, err := newDecoderDict(dictCap) + if err != nil { + return nil, err + } + r.d, err = newDecoder(ByteReader(lzma), state, dict, r.h.size) + if err != nil { + return nil, err + } + return r, nil +} + +// EOSMarker indicates that an EOS marker has been encountered. +func (r *Reader) EOSMarker() bool { + return r.d.eosMarker +} + +// Read returns uncompressed data. +func (r *Reader) Read(p []byte) (n int, err error) { + return r.d.Read(p) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader2.go b/vendor/github.com/ulikunitz/xz/lzma/reader2.go new file mode 100644 index 0000000..a55cfaa --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/reader2.go @@ -0,0 +1,232 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" + + "github.com/ulikunitz/xz/internal/xlog" +) + +// Reader2Config stores the parameters for the LZMA2 reader. +// format. +type Reader2Config struct { + DictCap int +} + +// fill converts the zero values of the configuration to the default values. +func (c *Reader2Config) fill() { + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } +} + +// Verify checks the reader configuration for errors. Zero configuration values +// will be replaced by default values. +func (c *Reader2Config) Verify() error { + c.fill() + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + return nil +} + +// Reader2 supports the reading of LZMA2 chunk sequences. Note that the +// first chunk should have a dictionary reset and the first compressed +// chunk a properties reset. The chunk sequence may not be terminated by +// an end-of-stream chunk. +type Reader2 struct { + r io.Reader + err error + + dict *decoderDict + ur *uncompressedReader + decoder *decoder + chunkReader io.Reader + + cstate chunkState + ctype chunkType +} + +// NewReader2 creates a reader for an LZMA2 chunk sequence. +func NewReader2(lzma2 io.Reader) (r *Reader2, err error) { + return Reader2Config{}.NewReader2(lzma2) +} + +// NewReader2 creates an LZMA2 reader using the given configuration. +func (c Reader2Config) NewReader2(lzma2 io.Reader) (r *Reader2, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + r = &Reader2{r: lzma2, cstate: start} + r.dict, err = newDecoderDict(c.DictCap) + if err != nil { + return nil, err + } + if err = r.startChunk(); err != nil { + r.err = err + } + return r, nil +} + +// uncompressed tests whether the chunk type specifies an uncompressed +// chunk. +func uncompressed(ctype chunkType) bool { + return ctype == cU || ctype == cUD +} + +// startChunk parses a new chunk. +func (r *Reader2) startChunk() error { + r.chunkReader = nil + header, err := readChunkHeader(r.r) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + xlog.Debugf("chunk header %v", header) + if err = r.cstate.next(header.ctype); err != nil { + return err + } + if r.cstate == stop { + return io.EOF + } + if header.ctype == cUD || header.ctype == cLRND { + r.dict.Reset() + } + size := int64(header.uncompressed) + 1 + if uncompressed(header.ctype) { + if r.ur != nil { + r.ur.Reopen(r.r, size) + } else { + r.ur = newUncompressedReader(r.r, r.dict, size) + } + r.chunkReader = r.ur + return nil + } + br := ByteReader(io.LimitReader(r.r, int64(header.compressed)+1)) + if r.decoder == nil { + state := newState(header.props) + r.decoder, err = newDecoder(br, state, r.dict, size) + if err != nil { + return err + } + r.chunkReader = r.decoder + return nil + } + switch header.ctype { + case cLR: + r.decoder.State.Reset() + case cLRN, cLRND: + r.decoder.State = newState(header.props) + } + err = r.decoder.Reopen(br, size) + if err != nil { + return err + } + r.chunkReader = r.decoder + return nil +} + +// Read reads data from the LZMA2 chunk sequence. +func (r *Reader2) Read(p []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + for n < len(p) { + var k int + k, err = r.chunkReader.Read(p[n:]) + n += k + if err != nil { + if err == io.EOF { + err = r.startChunk() + if err == nil { + continue + } + } + r.err = err + return n, err + } + if k == 0 { + r.err = errors.New("lzma: Reader2 doesn't get data") + return n, r.err + } + } + return n, nil +} + +// EOS returns whether the LZMA2 stream has been terminated by an +// end-of-stream chunk. +func (r *Reader2) EOS() bool { + return r.cstate == stop +} + +// uncompressedReader is used to read uncompressed chunks. +type uncompressedReader struct { + lr io.LimitedReader + Dict *decoderDict + eof bool + err error +} + +// newUncompressedReader initializes a new uncompressedReader. +func newUncompressedReader(r io.Reader, dict *decoderDict, size int64) *uncompressedReader { + ur := &uncompressedReader{ + lr: io.LimitedReader{R: r, N: size}, + Dict: dict, + } + return ur +} + +// Reopen reinitializes an uncompressed reader. +func (ur *uncompressedReader) Reopen(r io.Reader, size int64) { + ur.err = nil + ur.eof = false + ur.lr = io.LimitedReader{R: r, N: size} +} + +// fill reads uncompressed data into the dictionary. +func (ur *uncompressedReader) fill() error { + if !ur.eof { + n, err := io.CopyN(ur.Dict, &ur.lr, int64(ur.Dict.Available())) + if err != io.EOF { + return err + } + ur.eof = true + if n > 0 { + return nil + } + } + if ur.lr.N != 0 { + return io.ErrUnexpectedEOF + } + return io.EOF +} + +// Read reads uncompressed data from the limited reader. +func (ur *uncompressedReader) Read(p []byte) (n int, err error) { + if ur.err != nil { + return 0, ur.err + } + for { + var k int + k, err = ur.Dict.Read(p[n:]) + n += k + if n >= len(p) { + return n, nil + } + if err != nil { + break + } + err = ur.fill() + if err != nil { + break + } + } + ur.err = err + return n, err +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/state.go b/vendor/github.com/ulikunitz/xz/lzma/state.go new file mode 100644 index 0000000..5023510 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/state.go @@ -0,0 +1,151 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// states defines the overall state count +const states = 12 + +// State maintains the full state of the operation encoding or decoding +// process. +type state struct { + rep [4]uint32 + isMatch [states << maxPosBits]prob + isRepG0Long [states << maxPosBits]prob + isRep [states]prob + isRepG0 [states]prob + isRepG1 [states]prob + isRepG2 [states]prob + litCodec literalCodec + lenCodec lengthCodec + repLenCodec lengthCodec + distCodec distCodec + state uint32 + posBitMask uint32 + Properties Properties +} + +// initProbSlice initializes a slice of probabilities. +func initProbSlice(p []prob) { + for i := range p { + p[i] = probInit + } +} + +// Reset sets all state information to the original values. +func (s *state) Reset() { + p := s.Properties + *s = state{ + Properties: p, + // dict: s.dict, + posBitMask: (uint32(1) << uint(p.PB)) - 1, + } + initProbSlice(s.isMatch[:]) + initProbSlice(s.isRep[:]) + initProbSlice(s.isRepG0[:]) + initProbSlice(s.isRepG1[:]) + initProbSlice(s.isRepG2[:]) + initProbSlice(s.isRepG0Long[:]) + s.litCodec.init(p.LC, p.LP) + s.lenCodec.init() + s.repLenCodec.init() + s.distCodec.init() +} + +// initState initializes the state. +func initState(s *state, p Properties) { + *s = state{Properties: p} + s.Reset() +} + +// newState creates a new state from the give Properties. +func newState(p Properties) *state { + s := &state{Properties: p} + s.Reset() + return s +} + +// deepcopy initializes s as a deep copy of the source. +func (s *state) deepcopy(src *state) { + if s == src { + return + } + s.rep = src.rep + s.isMatch = src.isMatch + s.isRepG0Long = src.isRepG0Long + s.isRep = src.isRep + s.isRepG0 = src.isRepG0 + s.isRepG1 = src.isRepG1 + s.isRepG2 = src.isRepG2 + s.litCodec.deepcopy(&src.litCodec) + s.lenCodec.deepcopy(&src.lenCodec) + s.repLenCodec.deepcopy(&src.repLenCodec) + s.distCodec.deepcopy(&src.distCodec) + s.state = src.state + s.posBitMask = src.posBitMask + s.Properties = src.Properties +} + +// cloneState creates a new clone of the give state. +func cloneState(src *state) *state { + s := new(state) + s.deepcopy(src) + return s +} + +// updateStateLiteral updates the state for a literal. +func (s *state) updateStateLiteral() { + switch { + case s.state < 4: + s.state = 0 + return + case s.state < 10: + s.state -= 3 + return + } + s.state -= 6 +} + +// updateStateMatch updates the state for a match. +func (s *state) updateStateMatch() { + if s.state < 7 { + s.state = 7 + } else { + s.state = 10 + } +} + +// updateStateRep updates the state for a repetition. +func (s *state) updateStateRep() { + if s.state < 7 { + s.state = 8 + } else { + s.state = 11 + } +} + +// updateStateShortRep updates the state for a short repetition. +func (s *state) updateStateShortRep() { + if s.state < 7 { + s.state = 9 + } else { + s.state = 11 + } +} + +// states computes the states of the operation codec. +func (s *state) states(dictHead int64) (state1, state2, posState uint32) { + state1 = s.state + posState = uint32(dictHead) & s.posBitMask + state2 = (s.state << maxPosBits) | posState + return +} + +// litState computes the literal state. +func (s *state) litState(prev byte, dictHead int64) uint32 { + lp, lc := uint(s.Properties.LP), uint(s.Properties.LC) + litState := ((uint32(dictHead) & ((1 << lp) - 1)) << lc) | + (uint32(prev) >> (8 - lc)) + return litState +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go b/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go new file mode 100644 index 0000000..504b3d7 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go @@ -0,0 +1,133 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// treeCodec encodes or decodes values with a fixed bit size. It is using a +// tree of probability value. The root of the tree is the most-significant bit. +type treeCodec struct { + probTree +} + +// makeTreeCodec makes a tree codec. The bits value must be inside the range +// [1,32]. +func makeTreeCodec(bits int) treeCodec { + return treeCodec{makeProbTree(bits)} +} + +// deepcopy initializes tc as a deep copy of the source. +func (tc *treeCodec) deepcopy(src *treeCodec) { + tc.probTree.deepcopy(&src.probTree) +} + +// Encode uses the range encoder to encode a fixed-bit-size value. +func (tc *treeCodec) Encode(e *rangeEncoder, v uint32) (err error) { + m := uint32(1) + for i := int(tc.bits) - 1; i >= 0; i-- { + b := (v >> uint(i)) & 1 + if err := e.EncodeBit(b, &tc.probs[m]); err != nil { + return err + } + m = (m << 1) | b + } + return nil +} + +// Decodes uses the range decoder to decode a fixed-bit-size value. Errors may +// be caused by the range decoder. +func (tc *treeCodec) Decode(d *rangeDecoder) (v uint32, err error) { + m := uint32(1) + for j := 0; j < int(tc.bits); j++ { + b, err := d.DecodeBit(&tc.probs[m]) + if err != nil { + return 0, err + } + m = (m << 1) | b + } + return m - (1 << uint(tc.bits)), nil +} + +// treeReverseCodec is another tree codec, where the least-significant bit is +// the start of the probability tree. +type treeReverseCodec struct { + probTree +} + +// deepcopy initializes the treeReverseCodec as a deep copy of the +// source. +func (tc *treeReverseCodec) deepcopy(src *treeReverseCodec) { + tc.probTree.deepcopy(&src.probTree) +} + +// makeTreeReverseCodec creates treeReverseCodec value. The bits argument must +// be in the range [1,32]. +func makeTreeReverseCodec(bits int) treeReverseCodec { + return treeReverseCodec{makeProbTree(bits)} +} + +// Encode uses range encoder to encode a fixed-bit-size value. The range +// encoder may cause errors. +func (tc *treeReverseCodec) Encode(v uint32, e *rangeEncoder) (err error) { + m := uint32(1) + for i := uint(0); i < uint(tc.bits); i++ { + b := (v >> i) & 1 + if err := e.EncodeBit(b, &tc.probs[m]); err != nil { + return err + } + m = (m << 1) | b + } + return nil +} + +// Decodes uses the range decoder to decode a fixed-bit-size value. Errors +// returned by the range decoder will be returned. +func (tc *treeReverseCodec) Decode(d *rangeDecoder) (v uint32, err error) { + m := uint32(1) + for j := uint(0); j < uint(tc.bits); j++ { + b, err := d.DecodeBit(&tc.probs[m]) + if err != nil { + return 0, err + } + m = (m << 1) | b + v |= b << j + } + return v, nil +} + +// probTree stores enough probability values to be used by the treeEncode and +// treeDecode methods of the range coder types. +type probTree struct { + probs []prob + bits byte +} + +// deepcopy initializes the probTree value as a deep copy of the source. +func (t *probTree) deepcopy(src *probTree) { + if t == src { + return + } + t.probs = make([]prob, len(src.probs)) + copy(t.probs, src.probs) + t.bits = src.bits +} + +// makeProbTree initializes a probTree structure. +func makeProbTree(bits int) probTree { + if !(1 <= bits && bits <= 32) { + panic("bits outside of range [1,32]") + } + t := probTree{ + bits: byte(bits), + probs: make([]prob, 1< 0 { + c.SizeInHeader = true + } + if !c.SizeInHeader { + c.EOSMarker = true + } +} + +// Verify checks WriterConfig for errors. Verify will replace zero +// values with default values. +func (c *WriterConfig) Verify() error { + c.fill() + var err error + if c == nil { + return errors.New("lzma: WriterConfig is nil") + } + if c.Properties == nil { + return errors.New("lzma: WriterConfig has no Properties set") + } + if err = c.Properties.verify(); err != nil { + return err + } + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + if !(maxMatchLen <= c.BufSize) { + return errors.New("lzma: lookahead buffer size too small") + } + if c.SizeInHeader { + if c.Size < 0 { + return errors.New("lzma: negative size not supported") + } + } else if !c.EOSMarker { + return errors.New("lzma: EOS marker is required") + } + if err = c.Matcher.verify(); err != nil { + return err + } + + return nil +} + +// header returns the header structure for this configuration. +func (c *WriterConfig) header() header { + h := header{ + properties: *c.Properties, + dictCap: c.DictCap, + size: -1, + } + if c.SizeInHeader { + h.size = c.Size + } + return h +} + +// Writer writes an LZMA stream in the classic format. +type Writer struct { + h header + bw io.ByteWriter + buf *bufio.Writer + e *encoder +} + +// NewWriter creates a new LZMA writer for the classic format. The +// method will write the header to the underlying stream. +func (c WriterConfig) NewWriter(lzma io.Writer) (w *Writer, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + w = &Writer{h: c.header()} + + var ok bool + w.bw, ok = lzma.(io.ByteWriter) + if !ok { + w.buf = bufio.NewWriter(lzma) + w.bw = w.buf + } + state := newState(w.h.properties) + m, err := c.Matcher.new(w.h.dictCap) + if err != nil { + return nil, err + } + dict, err := newEncoderDict(w.h.dictCap, c.BufSize, m) + if err != nil { + return nil, err + } + var flags encoderFlags + if c.EOSMarker { + flags = eosMarker + } + if w.e, err = newEncoder(w.bw, state, dict, flags); err != nil { + return nil, err + } + + if err = w.writeHeader(); err != nil { + return nil, err + } + return w, nil +} + +// NewWriter creates a new LZMA writer using the classic format. The +// function writes the header to the underlying stream. +func NewWriter(lzma io.Writer) (w *Writer, err error) { + return WriterConfig{}.NewWriter(lzma) +} + +// writeHeader writes the LZMA header into the stream. +func (w *Writer) writeHeader() error { + data, err := w.h.marshalBinary() + if err != nil { + return err + } + _, err = w.bw.(io.Writer).Write(data) + return err +} + +// Write puts data into the Writer. +func (w *Writer) Write(p []byte) (n int, err error) { + if w.h.size >= 0 { + m := w.h.size + m -= w.e.Compressed() + int64(w.e.dict.Buffered()) + if m < 0 { + m = 0 + } + if m < int64(len(p)) { + p = p[:m] + err = ErrNoSpace + } + } + var werr error + if n, werr = w.e.Write(p); werr != nil { + err = werr + } + return n, err +} + +// Close closes the writer stream. It ensures that all data from the +// buffer will be compressed and the LZMA stream will be finished. +func (w *Writer) Close() error { + if w.h.size >= 0 { + n := w.e.Compressed() + int64(w.e.dict.Buffered()) + if n != w.h.size { + return errSize + } + } + err := w.e.Close() + if w.buf != nil { + ferr := w.buf.Flush() + if err == nil { + err = ferr + } + } + return err +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/writer2.go b/vendor/github.com/ulikunitz/xz/lzma/writer2.go new file mode 100644 index 0000000..7c1afe1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/writer2.go @@ -0,0 +1,305 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "bytes" + "errors" + "io" +) + +// Writer2Config is used to create a Writer2 using parameters. +type Writer2Config struct { + // The properties for the encoding. If the it is nil the value + // {LC: 3, LP: 0, PB: 2} will be chosen. + Properties *Properties + // The capacity of the dictionary. If DictCap is zero, the value + // 8 MiB will be chosen. + DictCap int + // Size of the lookahead buffer; value 0 indicates default size + // 4096 + BufSize int + // Match algorithm + Matcher MatchAlgorithm +} + +// fill replaces zero values with default values. +func (c *Writer2Config) fill() { + if c.Properties == nil { + c.Properties = &Properties{LC: 3, LP: 0, PB: 2} + } + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } + if c.BufSize == 0 { + c.BufSize = 4096 + } +} + +// Verify checks the Writer2Config for correctness. Zero values will be +// replaced by default values. +func (c *Writer2Config) Verify() error { + c.fill() + var err error + if c == nil { + return errors.New("lzma: WriterConfig is nil") + } + if c.Properties == nil { + return errors.New("lzma: WriterConfig has no Properties set") + } + if err = c.Properties.verify(); err != nil { + return err + } + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + if !(maxMatchLen <= c.BufSize) { + return errors.New("lzma: lookahead buffer size too small") + } + if c.Properties.LC+c.Properties.LP > 4 { + return errors.New("lzma: sum of lc and lp exceeds 4") + } + if err = c.Matcher.verify(); err != nil { + return err + } + return nil +} + +// Writer2 supports the creation of an LZMA2 stream. But note that +// written data is buffered, so call Flush or Close to write data to the +// underlying writer. The Close method writes the end-of-stream marker +// to the stream. So you may be able to concatenate the output of two +// writers as long the output of the first writer has only been flushed +// but not closed. +// +// Any change to the fields Properties, DictCap must be done before the +// first call to Write, Flush or Close. +type Writer2 struct { + w io.Writer + + start *state + encoder *encoder + + cstate chunkState + ctype chunkType + + buf bytes.Buffer + lbw LimitedByteWriter +} + +// NewWriter2 creates an LZMA2 chunk sequence writer with the default +// parameters and options. +func NewWriter2(lzma2 io.Writer) (w *Writer2, err error) { + return Writer2Config{}.NewWriter2(lzma2) +} + +// NewWriter2 creates a new LZMA2 writer using the given configuration. +func (c Writer2Config) NewWriter2(lzma2 io.Writer) (w *Writer2, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + w = &Writer2{ + w: lzma2, + start: newState(*c.Properties), + cstate: start, + ctype: start.defaultChunkType(), + } + w.buf.Grow(maxCompressed) + w.lbw = LimitedByteWriter{BW: &w.buf, N: maxCompressed} + m, err := c.Matcher.new(c.DictCap) + if err != nil { + return nil, err + } + d, err := newEncoderDict(c.DictCap, c.BufSize, m) + if err != nil { + return nil, err + } + w.encoder, err = newEncoder(&w.lbw, cloneState(w.start), d, 0) + if err != nil { + return nil, err + } + return w, nil +} + +// written returns the number of bytes written to the current chunk +func (w *Writer2) written() int { + if w.encoder == nil { + return 0 + } + return int(w.encoder.Compressed()) + w.encoder.dict.Buffered() +} + +// errClosed indicates that the writer is closed. +var errClosed = errors.New("lzma: writer closed") + +// Writes data to LZMA2 stream. Note that written data will be buffered. +// Use Flush or Close to ensure that data is written to the underlying +// writer. +func (w *Writer2) Write(p []byte) (n int, err error) { + if w.cstate == stop { + return 0, errClosed + } + for n < len(p) { + m := maxUncompressed - w.written() + if m <= 0 { + panic("lzma: maxUncompressed reached") + } + var q []byte + if n+m < len(p) { + q = p[n : n+m] + } else { + q = p[n:] + } + k, err := w.encoder.Write(q) + n += k + if err != nil && err != ErrLimit { + return n, err + } + if err == ErrLimit || k == m { + if err = w.flushChunk(); err != nil { + return n, err + } + } + } + return n, nil +} + +// writeUncompressedChunk writes an uncompressed chunk to the LZMA2 +// stream. +func (w *Writer2) writeUncompressedChunk() error { + u := w.encoder.Compressed() + if u <= 0 { + return errors.New("lzma: can't write empty uncompressed chunk") + } + if u > maxUncompressed { + panic("overrun of uncompressed data limit") + } + switch w.ctype { + case cLRND: + w.ctype = cUD + default: + w.ctype = cU + } + w.encoder.state = w.start + + header := chunkHeader{ + ctype: w.ctype, + uncompressed: uint32(u - 1), + } + hdata, err := header.MarshalBinary() + if err != nil { + return err + } + if _, err = w.w.Write(hdata); err != nil { + return err + } + _, err = w.encoder.dict.CopyN(w.w, int(u)) + return err +} + +// writeCompressedChunk writes a compressed chunk to the underlying +// writer. +func (w *Writer2) writeCompressedChunk() error { + if w.ctype == cU || w.ctype == cUD { + panic("chunk type uncompressed") + } + + u := w.encoder.Compressed() + if u <= 0 { + return errors.New("writeCompressedChunk: empty chunk") + } + if u > maxUncompressed { + panic("overrun of uncompressed data limit") + } + c := w.buf.Len() + if c <= 0 { + panic("no compressed data") + } + if c > maxCompressed { + panic("overrun of compressed data limit") + } + header := chunkHeader{ + ctype: w.ctype, + uncompressed: uint32(u - 1), + compressed: uint16(c - 1), + props: w.encoder.state.Properties, + } + hdata, err := header.MarshalBinary() + if err != nil { + return err + } + if _, err = w.w.Write(hdata); err != nil { + return err + } + _, err = io.Copy(w.w, &w.buf) + return err +} + +// writes a single chunk to the underlying writer. +func (w *Writer2) writeChunk() error { + u := int(uncompressedHeaderLen + w.encoder.Compressed()) + c := headerLen(w.ctype) + w.buf.Len() + if u < c { + return w.writeUncompressedChunk() + } + return w.writeCompressedChunk() +} + +// flushChunk terminates the current chunk. The encoder will be reset +// to support the next chunk. +func (w *Writer2) flushChunk() error { + if w.written() == 0 { + return nil + } + var err error + if err = w.encoder.Close(); err != nil { + return err + } + if err = w.writeChunk(); err != nil { + return err + } + w.buf.Reset() + w.lbw.N = maxCompressed + if err = w.encoder.Reopen(&w.lbw); err != nil { + return err + } + if err = w.cstate.next(w.ctype); err != nil { + return err + } + w.ctype = w.cstate.defaultChunkType() + w.start = cloneState(w.encoder.state) + return nil +} + +// Flush writes all buffered data out to the underlying stream. This +// could result in multiple chunks to be created. +func (w *Writer2) Flush() error { + if w.cstate == stop { + return errClosed + } + for w.written() > 0 { + if err := w.flushChunk(); err != nil { + return err + } + } + return nil +} + +// Close terminates the LZMA2 stream with an EOS chunk. +func (w *Writer2) Close() error { + if w.cstate == stop { + return errClosed + } + if err := w.Flush(); err != nil { + return nil + } + // write zero byte EOS chunk + _, err := w.w.Write([]byte{0}) + if err != nil { + return err + } + w.cstate = stop + return nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzmafilter.go b/vendor/github.com/ulikunitz/xz/lzmafilter.go new file mode 100644 index 0000000..69cf5f7 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzmafilter.go @@ -0,0 +1,117 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "errors" + "fmt" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// LZMA filter constants. +const ( + lzmaFilterID = 0x21 + lzmaFilterLen = 3 +) + +// lzmaFilter declares the LZMA2 filter information stored in an xz +// block header. +type lzmaFilter struct { + dictCap int64 +} + +// String returns a representation of the LZMA filter. +func (f lzmaFilter) String() string { + return fmt.Sprintf("LZMA dict cap %#x", f.dictCap) +} + +// id returns the ID for the LZMA2 filter. +func (f lzmaFilter) id() uint64 { return lzmaFilterID } + +// MarshalBinary converts the lzmaFilter in its encoded representation. +func (f lzmaFilter) MarshalBinary() (data []byte, err error) { + c := lzma.EncodeDictCap(f.dictCap) + return []byte{lzmaFilterID, 1, c}, nil +} + +// UnmarshalBinary unmarshals the given data representation of the LZMA2 +// filter. +func (f *lzmaFilter) UnmarshalBinary(data []byte) error { + if len(data) != lzmaFilterLen { + return errors.New("xz: data for LZMA2 filter has wrong length") + } + if data[0] != lzmaFilterID { + return errors.New("xz: wrong LZMA2 filter id") + } + if data[1] != 1 { + return errors.New("xz: wrong LZMA2 filter size") + } + dc, err := lzma.DecodeDictCap(data[2]) + if err != nil { + return errors.New("xz: wrong LZMA2 dictionary size property") + } + + f.dictCap = dc + return nil +} + +// reader creates a new reader for the LZMA2 filter. +func (f lzmaFilter) reader(r io.Reader, c *ReaderConfig) (fr io.Reader, + err error) { + + config := new(lzma.Reader2Config) + if c != nil { + config.DictCap = c.DictCap + } + dc := int(f.dictCap) + if dc < 1 { + return nil, errors.New("xz: LZMA2 filter parameter " + + "dictionary capacity overflow") + } + if dc > config.DictCap { + config.DictCap = dc + } + + fr, err = config.NewReader2(r) + if err != nil { + return nil, err + } + return fr, nil +} + +// writeCloser creates a io.WriteCloser for the LZMA2 filter. +func (f lzmaFilter) writeCloser(w io.WriteCloser, c *WriterConfig, +) (fw io.WriteCloser, err error) { + config := new(lzma.Writer2Config) + if c != nil { + *config = lzma.Writer2Config{ + Properties: c.Properties, + DictCap: c.DictCap, + BufSize: c.BufSize, + Matcher: c.Matcher, + } + } + + dc := int(f.dictCap) + if dc < 1 { + return nil, errors.New("xz: LZMA2 filter parameter " + + "dictionary capacity overflow") + } + if dc > config.DictCap { + config.DictCap = dc + } + + fw, err = config.NewWriter2(w) + if err != nil { + return nil, err + } + return fw, nil +} + +// last returns true, because an LZMA2 filter must be the last filter in +// the filter list. +func (f lzmaFilter) last() bool { return true } diff --git a/vendor/github.com/ulikunitz/xz/make-docs b/vendor/github.com/ulikunitz/xz/make-docs new file mode 100755 index 0000000..a8c612c --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/make-docs @@ -0,0 +1,5 @@ +#!/bin/sh + +set -x +pandoc -t html5 -f markdown -s --css=doc/md.css -o README.html README.md +pandoc -t html5 -f markdown -s --css=doc/md.css -o TODO.html TODO.md diff --git a/vendor/github.com/ulikunitz/xz/reader.go b/vendor/github.com/ulikunitz/xz/reader.go new file mode 100644 index 0000000..0634c6b --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/reader.go @@ -0,0 +1,373 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xz supports the compression and decompression of xz files. It +// supports version 1.0.4 of the specification without the non-LZMA2 +// filters. See http://tukaani.org/xz/xz-file-format-1.0.4.txt +package xz + +import ( + "bytes" + "errors" + "fmt" + "hash" + "io" + + "github.com/ulikunitz/xz/internal/xlog" + "github.com/ulikunitz/xz/lzma" +) + +// ReaderConfig defines the parameters for the xz reader. The +// SingleStream parameter requests the reader to assume that the +// underlying stream contains only a single stream. +type ReaderConfig struct { + DictCap int + SingleStream bool +} + +// fill replaces all zero values with their default values. +func (c *ReaderConfig) fill() { + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } +} + +// Verify checks the reader parameters for Validity. Zero values will be +// replaced by default values. +func (c *ReaderConfig) Verify() error { + if c == nil { + return errors.New("xz: reader parameters are nil") + } + lc := lzma.Reader2Config{DictCap: c.DictCap} + if err := lc.Verify(); err != nil { + return err + } + return nil +} + +// Reader supports the reading of one or multiple xz streams. +type Reader struct { + ReaderConfig + + xz io.Reader + sr *streamReader +} + +// streamReader decodes a single xz stream +type streamReader struct { + ReaderConfig + + xz io.Reader + br *blockReader + newHash func() hash.Hash + h header + index []record +} + +// NewReader creates a new xz reader using the default parameters. +// The function reads and checks the header of the first XZ stream. The +// reader will process multiple streams including padding. +func NewReader(xz io.Reader) (r *Reader, err error) { + return ReaderConfig{}.NewReader(xz) +} + +// NewReader creates an xz stream reader. The created reader will be +// able to process multiple streams and padding unless a SingleStream +// has been set in the reader configuration c. +func (c ReaderConfig) NewReader(xz io.Reader) (r *Reader, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + r = &Reader{ + ReaderConfig: c, + xz: xz, + } + if r.sr, err = c.newStreamReader(xz); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + return r, nil +} + +var errUnexpectedData = errors.New("xz: unexpected data after stream") + +// Read reads uncompressed data from the stream. +func (r *Reader) Read(p []byte) (n int, err error) { + for n < len(p) { + if r.sr == nil { + if r.SingleStream { + data := make([]byte, 1) + _, err = io.ReadFull(r.xz, data) + if err != io.EOF { + return n, errUnexpectedData + } + return n, io.EOF + } + for { + r.sr, err = r.ReaderConfig.newStreamReader(r.xz) + if err != errPadding { + break + } + } + if err != nil { + return n, err + } + } + k, err := r.sr.Read(p[n:]) + n += k + if err != nil { + if err == io.EOF { + r.sr = nil + continue + } + return n, err + } + } + return n, nil +} + +var errPadding = errors.New("xz: padding (4 zero bytes) encountered") + +// newStreamReader creates a new xz stream reader using the given configuration +// parameters. NewReader reads and checks the header of the xz stream. +func (c ReaderConfig) newStreamReader(xz io.Reader) (r *streamReader, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + data := make([]byte, HeaderLen) + if _, err := io.ReadFull(xz, data[:4]); err != nil { + return nil, err + } + if bytes.Equal(data[:4], []byte{0, 0, 0, 0}) { + return nil, errPadding + } + if _, err = io.ReadFull(xz, data[4:]); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + r = &streamReader{ + ReaderConfig: c, + xz: xz, + index: make([]record, 0, 4), + } + if err = r.h.UnmarshalBinary(data); err != nil { + return nil, err + } + xlog.Debugf("xz header %s", r.h) + if r.newHash, err = newHashFunc(r.h.flags); err != nil { + return nil, err + } + return r, nil +} + +// errIndex indicates an error with the xz file index. +var errIndex = errors.New("xz: error in xz file index") + +// readTail reads the index body and the xz footer. +func (r *streamReader) readTail() error { + index, n, err := readIndexBody(r.xz) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + if len(index) != len(r.index) { + return fmt.Errorf("xz: index length is %d; want %d", + len(index), len(r.index)) + } + for i, rec := range r.index { + if rec != index[i] { + return fmt.Errorf("xz: record %d is %v; want %v", + i, rec, index[i]) + } + } + + p := make([]byte, footerLen) + if _, err = io.ReadFull(r.xz, p); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + var f footer + if err = f.UnmarshalBinary(p); err != nil { + return err + } + xlog.Debugf("xz footer %s", f) + if f.flags != r.h.flags { + return errors.New("xz: footer flags incorrect") + } + if f.indexSize != int64(n)+1 { + return errors.New("xz: index size in footer wrong") + } + return nil +} + +// Read reads actual data from the xz stream. +func (r *streamReader) Read(p []byte) (n int, err error) { + for n < len(p) { + if r.br == nil { + bh, hlen, err := readBlockHeader(r.xz) + if err != nil { + if err == errIndexIndicator { + if err = r.readTail(); err != nil { + return n, err + } + return n, io.EOF + } + return n, err + } + xlog.Debugf("block %v", *bh) + r.br, err = r.ReaderConfig.newBlockReader(r.xz, bh, + hlen, r.newHash()) + if err != nil { + return n, err + } + } + k, err := r.br.Read(p[n:]) + n += k + if err != nil { + if err == io.EOF { + r.index = append(r.index, r.br.record()) + r.br = nil + } else { + return n, err + } + } + } + return n, nil +} + +// countingReader is a reader that counts the bytes read. +type countingReader struct { + r io.Reader + n int64 +} + +// Read reads data from the wrapped reader and adds it to the n field. +func (lr *countingReader) Read(p []byte) (n int, err error) { + n, err = lr.r.Read(p) + lr.n += int64(n) + return n, err +} + +// blockReader supports the reading of a block. +type blockReader struct { + lxz countingReader + header *blockHeader + headerLen int + n int64 + hash hash.Hash + r io.Reader + err error +} + +// newBlockReader creates a new block reader. +func (c *ReaderConfig) newBlockReader(xz io.Reader, h *blockHeader, + hlen int, hash hash.Hash) (br *blockReader, err error) { + + br = &blockReader{ + lxz: countingReader{r: xz}, + header: h, + headerLen: hlen, + hash: hash, + } + + fr, err := c.newFilterReader(&br.lxz, h.filters) + if err != nil { + return nil, err + } + br.r = io.TeeReader(fr, br.hash) + + return br, nil +} + +// uncompressedSize returns the uncompressed size of the block. +func (br *blockReader) uncompressedSize() int64 { + return br.n +} + +// compressedSize returns the compressed size of the block. +func (br *blockReader) compressedSize() int64 { + return br.lxz.n +} + +// unpaddedSize computes the unpadded size for the block. +func (br *blockReader) unpaddedSize() int64 { + n := int64(br.headerLen) + n += br.compressedSize() + n += int64(br.hash.Size()) + return n +} + +// record returns the index record for the current block. +func (br *blockReader) record() record { + return record{br.unpaddedSize(), br.uncompressedSize()} +} + +// errBlockSize indicates that the size of the block in the block header +// is wrong. +var errBlockSize = errors.New("xz: wrong uncompressed size for block") + +// Read reads data from the block. +func (br *blockReader) Read(p []byte) (n int, err error) { + n, err = br.r.Read(p) + br.n += int64(n) + + u := br.header.uncompressedSize + if u >= 0 && br.uncompressedSize() > u { + return n, errors.New("xz: wrong uncompressed size for block") + } + c := br.header.compressedSize + if c >= 0 && br.compressedSize() > c { + return n, errors.New("xz: wrong compressed size for block") + } + if err != io.EOF { + return n, err + } + if br.uncompressedSize() < u || br.compressedSize() < c { + return n, io.ErrUnexpectedEOF + } + + s := br.hash.Size() + k := padLen(br.lxz.n) + q := make([]byte, k+s, k+2*s) + if _, err = io.ReadFull(br.lxz.r, q); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return n, err + } + if !allZeros(q[:k]) { + return n, errors.New("xz: non-zero block padding") + } + checkSum := q[k:] + computedSum := br.hash.Sum(checkSum[s:]) + if !bytes.Equal(checkSum, computedSum) { + return n, errors.New("xz: checksum error for block") + } + return n, io.EOF +} + +func (c *ReaderConfig) newFilterReader(r io.Reader, f []filter) (fr io.Reader, + err error) { + + if err = verifyFilters(f); err != nil { + return nil, err + } + + fr = r + for i := len(f) - 1; i >= 0; i-- { + fr, err = f[i].reader(fr, c) + if err != nil { + return nil, err + } + } + return fr, nil +} diff --git a/vendor/github.com/ulikunitz/xz/writer.go b/vendor/github.com/ulikunitz/xz/writer.go new file mode 100644 index 0000000..c126f70 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/writer.go @@ -0,0 +1,386 @@ +// Copyright 2014-2017 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "errors" + "hash" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// WriterConfig describe the parameters for an xz writer. +type WriterConfig struct { + Properties *lzma.Properties + DictCap int + BufSize int + BlockSize int64 + // checksum method: CRC32, CRC64 or SHA256 + CheckSum byte + // match algorithm + Matcher lzma.MatchAlgorithm +} + +// fill replaces zero values with default values. +func (c *WriterConfig) fill() { + if c.Properties == nil { + c.Properties = &lzma.Properties{LC: 3, LP: 0, PB: 2} + } + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } + if c.BufSize == 0 { + c.BufSize = 4096 + } + if c.BlockSize == 0 { + c.BlockSize = maxInt64 + } + if c.CheckSum == 0 { + c.CheckSum = CRC64 + } +} + +// Verify checks the configuration for errors. Zero values will be +// replaced by default values. +func (c *WriterConfig) Verify() error { + if c == nil { + return errors.New("xz: writer configuration is nil") + } + c.fill() + lc := lzma.Writer2Config{ + Properties: c.Properties, + DictCap: c.DictCap, + BufSize: c.BufSize, + Matcher: c.Matcher, + } + if err := lc.Verify(); err != nil { + return err + } + if c.BlockSize <= 0 { + return errors.New("xz: block size out of range") + } + if err := verifyFlags(c.CheckSum); err != nil { + return err + } + return nil +} + +// filters creates the filter list for the given parameters. +func (c *WriterConfig) filters() []filter { + return []filter{&lzmaFilter{int64(c.DictCap)}} +} + +// maxInt64 defines the maximum 64-bit signed integer. +const maxInt64 = 1<<63 - 1 + +// verifyFilters checks the filter list for the length and the right +// sequence of filters. +func verifyFilters(f []filter) error { + if len(f) == 0 { + return errors.New("xz: no filters") + } + if len(f) > 4 { + return errors.New("xz: more than four filters") + } + for _, g := range f[:len(f)-1] { + if g.last() { + return errors.New("xz: last filter is not last") + } + } + if !f[len(f)-1].last() { + return errors.New("xz: wrong last filter") + } + return nil +} + +// newFilterWriteCloser converts a filter list into a WriteCloser that +// can be used by a blockWriter. +func (c *WriterConfig) newFilterWriteCloser(w io.Writer, f []filter) (fw io.WriteCloser, err error) { + if err = verifyFilters(f); err != nil { + return nil, err + } + fw = nopWriteCloser(w) + for i := len(f) - 1; i >= 0; i-- { + fw, err = f[i].writeCloser(fw, c) + if err != nil { + return nil, err + } + } + return fw, nil +} + +// nopWCloser implements a WriteCloser with a Close method not doing +// anything. +type nopWCloser struct { + io.Writer +} + +// Close returns nil and doesn't do anything else. +func (c nopWCloser) Close() error { + return nil +} + +// nopWriteCloser converts the Writer into a WriteCloser with a Close +// function that does nothing beside returning nil. +func nopWriteCloser(w io.Writer) io.WriteCloser { + return nopWCloser{w} +} + +// Writer compresses data written to it. It is an io.WriteCloser. +type Writer struct { + WriterConfig + + xz io.Writer + bw *blockWriter + newHash func() hash.Hash + h header + index []record + closed bool +} + +// newBlockWriter creates a new block writer writes the header out. +func (w *Writer) newBlockWriter() error { + var err error + w.bw, err = w.WriterConfig.newBlockWriter(w.xz, w.newHash()) + if err != nil { + return err + } + if err = w.bw.writeHeader(w.xz); err != nil { + return err + } + return nil +} + +// closeBlockWriter closes a block writer and records the sizes in the +// index. +func (w *Writer) closeBlockWriter() error { + var err error + if err = w.bw.Close(); err != nil { + return err + } + w.index = append(w.index, w.bw.record()) + return nil +} + +// NewWriter creates a new xz writer using default parameters. +func NewWriter(xz io.Writer) (w *Writer, err error) { + return WriterConfig{}.NewWriter(xz) +} + +// NewWriter creates a new Writer using the given configuration parameters. +func (c WriterConfig) NewWriter(xz io.Writer) (w *Writer, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + w = &Writer{ + WriterConfig: c, + xz: xz, + h: header{c.CheckSum}, + index: make([]record, 0, 4), + } + if w.newHash, err = newHashFunc(c.CheckSum); err != nil { + return nil, err + } + data, err := w.h.MarshalBinary() + if _, err = xz.Write(data); err != nil { + return nil, err + } + if err = w.newBlockWriter(); err != nil { + return nil, err + } + return w, nil + +} + +// Write compresses the uncompressed data provided. +func (w *Writer) Write(p []byte) (n int, err error) { + if w.closed { + return 0, errClosed + } + for { + k, err := w.bw.Write(p[n:]) + n += k + if err != errNoSpace { + return n, err + } + if err = w.closeBlockWriter(); err != nil { + return n, err + } + if err = w.newBlockWriter(); err != nil { + return n, err + } + } +} + +// Close closes the writer and adds the footer to the Writer. Close +// doesn't close the underlying writer. +func (w *Writer) Close() error { + if w.closed { + return errClosed + } + w.closed = true + var err error + if err = w.closeBlockWriter(); err != nil { + return err + } + + f := footer{flags: w.h.flags} + if f.indexSize, err = writeIndex(w.xz, w.index); err != nil { + return err + } + data, err := f.MarshalBinary() + if err != nil { + return err + } + if _, err = w.xz.Write(data); err != nil { + return err + } + return nil +} + +// countingWriter is a writer that counts all data written to it. +type countingWriter struct { + w io.Writer + n int64 +} + +// Write writes data to the countingWriter. +func (cw *countingWriter) Write(p []byte) (n int, err error) { + n, err = cw.w.Write(p) + cw.n += int64(n) + if err == nil && cw.n < 0 { + return n, errors.New("xz: counter overflow") + } + return +} + +// blockWriter is writes a single block. +type blockWriter struct { + cxz countingWriter + // mw combines io.WriteCloser w and the hash. + mw io.Writer + w io.WriteCloser + n int64 + blockSize int64 + closed bool + headerLen int + + filters []filter + hash hash.Hash +} + +// newBlockWriter creates a new block writer. +func (c *WriterConfig) newBlockWriter(xz io.Writer, hash hash.Hash) (bw *blockWriter, err error) { + bw = &blockWriter{ + cxz: countingWriter{w: xz}, + blockSize: c.BlockSize, + filters: c.filters(), + hash: hash, + } + bw.w, err = c.newFilterWriteCloser(&bw.cxz, bw.filters) + if err != nil { + return nil, err + } + bw.mw = io.MultiWriter(bw.w, bw.hash) + return bw, nil +} + +// writeHeader writes the header. If the function is called after Close +// the commpressedSize and uncompressedSize fields will be filled. +func (bw *blockWriter) writeHeader(w io.Writer) error { + h := blockHeader{ + compressedSize: -1, + uncompressedSize: -1, + filters: bw.filters, + } + if bw.closed { + h.compressedSize = bw.compressedSize() + h.uncompressedSize = bw.uncompressedSize() + } + data, err := h.MarshalBinary() + if err != nil { + return err + } + if _, err = w.Write(data); err != nil { + return err + } + bw.headerLen = len(data) + return nil +} + +// compressed size returns the amount of data written to the underlying +// stream. +func (bw *blockWriter) compressedSize() int64 { + return bw.cxz.n +} + +// uncompressedSize returns the number of data written to the +// blockWriter +func (bw *blockWriter) uncompressedSize() int64 { + return bw.n +} + +// unpaddedSize returns the sum of the header length, the uncompressed +// size of the block and the hash size. +func (bw *blockWriter) unpaddedSize() int64 { + if bw.headerLen <= 0 { + panic("xz: block header not written") + } + n := int64(bw.headerLen) + n += bw.compressedSize() + n += int64(bw.hash.Size()) + return n +} + +// record returns the record for the current stream. Call Close before +// calling this method. +func (bw *blockWriter) record() record { + return record{bw.unpaddedSize(), bw.uncompressedSize()} +} + +var errClosed = errors.New("xz: writer already closed") + +var errNoSpace = errors.New("xz: no space") + +// Write writes uncompressed data to the block writer. +func (bw *blockWriter) Write(p []byte) (n int, err error) { + if bw.closed { + return 0, errClosed + } + + t := bw.blockSize - bw.n + if int64(len(p)) > t { + err = errNoSpace + p = p[:t] + } + + var werr error + n, werr = bw.mw.Write(p) + bw.n += int64(n) + if werr != nil { + return n, werr + } + return n, err +} + +// Close closes the writer. +func (bw *blockWriter) Close() error { + if bw.closed { + return errClosed + } + bw.closed = true + if err := bw.w.Close(); err != nil { + return err + } + s := bw.hash.Size() + k := padLen(bw.cxz.n) + p := make([]byte, k+s) + bw.hash.Sum(p[k:k]) + if _, err := bw.cxz.w.Write(p); err != nil { + return err + } + return nil +}