support Incremental upload (#8)

feat: support incremental upload

#7

.env

@@ -1,10 +1,11 @@
 ENDPOINT = fangbinwei-blog.oss-cn-shanghai.aliyuncs.com
-FOLDER = example
+FOLDER = testdata/group1
 BUCKET = fangbinwei-blog
 INDEX_PAGE = index.html
 NOT_FOUND_PAGE = 404.html
 CNAME = true
 HTML_CACHE_CONTROL = no-cache
 IMAGE_CACHE_CONTROL = max-age=864000
 OTHER_CACHE_CONTROL = max-age=2592000
-SKIP_SETTING = false
+SKIP_SETTING = false
+INCREMENTAL = true

.github/workflows/test.yml

@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - master
+      - 'master'
 
 jobs:
   build:
@@ -41,7 +41,7 @@ jobs:
     - run: yarn install
     - run: yarn docs:build
     - name: upload files to OSS
-      uses: fangbinwei/aliyun-oss-website-action@v1
+      uses: fangbinwei/aliyun-oss-website-action@master
       with:
           accessKeyId: ${{ secrets.ACCESS_KEY_ID }}
           accessKeySecret: ${{ secrets.ACCESS_KEY_SECRET }}
@@ -61,3 +61,29 @@ jobs:
             demo2/*.md
             demo2/*/*.md
 
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: fangbinwei/aliyun-oss-website-action:latest

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13
+FROM golang:1.16
 
 WORKDIR /go/src/github.com/fangbinwei/aliyun-oss-website-action
 COPY . .

README.md

@@ -7,7 +7,7 @@ deploy website on aliyun OSS(Alibaba Cloud OSS)
 ## 概览
 - 在阿里云OSS创建一个存放网站的bucket
 - 准备一个域名, 可能需要备案(bucket选择非大陆区域, 可以不备案, 但是如果CDN加速区域包括大陆, 仍然需要备案)
-- 在你的网站repo中, 配置github action, 当action触发, 会清除bucket中原有的文件, 上传网站repo生成的资源文件到bucket中
+- 在你的网站repo中, 配置github action, action 触发则增量上传网站repo生成的资源文件到bucket中
 - 通过阿里云OSS的CDN, 可以很方便地加速网站的访问, 支持HTTPS
 
 ## Usage
@@ -32,13 +32,28 @@ deploy website on aliyun OSS(Alibaba Cloud OSS)
 - `bucket`: **必填**,部署网站的bucket, 用于存放网站的资源
 - `indexPage`: 默认`index.html`.网站首页(用于[静态页面配置](#静态页面配置))
 - `notFoundPage`: 默认`404.html`.网站404页面(用于[静态页面配置](#静态页面配置))
+- `incremental`: 默认`true`. 使用增量上传.
 - `skipSetting`: 默认`false`, 是否跳过设置[静态页面配置](#静态页面配置)
 - `htmlCacheControl`: 默认`no-cache`
 - `imageCacheControl`: 默认`max-age=864000`
 - `otherCacheControl`: 默认`max-age=2592000`
 - `exclude`: 不上传`folder`下的某些文件/文件夹
 - `cname`: 默认`false`. 若`endpoint`填写自定义域名/bucket域名, 需设置为`true`. (使用CDN的场景下, 不推荐使用自定义域名)
 
+## incremental
+**开启`incremental`**
+上传文件到OSS后, 还会将文件的`ContentMD5`和`Cache-Control`收集到名为`.actioninfo`的私有文件中. 当再次触发action的时候, 会将待上传的文件信息与`.actioninfo`中记录的信息比对, 信息未发生变化的文件将跳过上传步骤, 只进行增量上传. 且在上传之后, 根据`.actioninfo`和已上传的文件信息, 将OSS中多余的文件进行删除.
+
+> `.actioninfo` 记录了上一次action执行时, 所上传的文件信息. 私有, 不可公共读写.
+
+**关闭`incremental`** 或 OSS中不存在`.actioninfo`文件
+
+会执行如下步骤
+1. 清除所有OSS中已有的文件
+2. 上传新的文件到OSS中
+
+> **计划未来优化这个步骤, 优化后, 先上传新的文件到OSS中, 再diff删除多余的文件.** 
+
 ## Cache-Control
 为上传的资源默认设置的`Cache-Control`如下
 |资源类型 | Cache-Control|

action.yml

@@ -31,6 +31,10 @@ inputs:
     description: '`true` to skip setting static pages related configuration. `indexPage`, `notFoundPage` will not be used.'
     required: false
     default: 'false'
+  incremental:
+    description: 'Save info of uploaded files to increase next upload speed'
+    required: false
+    default: 'true'
   indexPage:
     description: 'index page'
     required: false
@@ -63,6 +67,7 @@ runs:
     FOLDER: ${{ inputs.folder }}
     EXCLUDE: ${{ inputs.exclude }}
     SKIP_SETTING: ${{ inputs.skipSetting }}
+    INCREMENTAL: ${{ inputs.incremental }}
     INDEX_PAGE: ${{ inputs.indexPage }}
     NOT_FOUND_PAGE: ${{ inputs.notFoundPage }}
     HTML_CACHE_CONTROL: ${{ inputs.htmlCacheControl }}

config/config.go

@@ -21,6 +21,7 @@ var (
 	Client          *oss.Client
 	Bucket          *oss.Bucket
 	SkipSetting     bool
+	IsIncremental     bool
 
 	IndexPage         string
 	NotFoundPage      string
@@ -41,6 +42,7 @@ func init() {
 	Exclude = utils.GetActionInputAsSlice(os.Getenv("EXCLUDE"))
 	BucketName = os.Getenv("BUCKET")
 	SkipSetting = os.Getenv("SKIP_SETTING") == "true"
+	IsIncremental = os.Getenv("INCREMENTAL") == "true"
 
 	IndexPage = utils.Getenv("INDEX_PAGE", "index.html")
 	NotFoundPage = utils.Getenv("NOT_FOUND_PAGE", "404.html")
@@ -53,8 +55,8 @@ func init() {
 		fmt.Println(err)
 	}
 	fmt.Printf("current directory: %s\n", currentPath)
-	fmt.Printf("endpoint: %s\nbucketName: %s\nfolder: %s\nexclude: %v\nindexPage: %s\nnotFoundPage: %s\nisCname: %t\nskipSetting: %t\n",
-		Endpoint, BucketName, Folder, Exclude, IndexPage, NotFoundPage, IsCname, SkipSetting)
+	fmt.Printf("endpoint: %s\nbucketName: %s\nfolder: %s\nincremental: %t\nexclude: %v\nindexPage: %s\nnotFoundPage: %s\nisCname: %t\nskipSetting: %t\n",
+		Endpoint, BucketName, Folder, IsIncremental, Exclude, IndexPage, NotFoundPage, IsCname, SkipSetting)
 	fmt.Printf("HTMLCacheControl: %s\nimageCacheControl: %s\notherCacheControl: %s\n",
 		HTMLCacheControl, ImageCacheControl, OtherCacheControl)
 

main.go

@@ -18,17 +18,44 @@ func main() {
 		fmt.Println("skip setting static pages related configuration")
 	}
 
-	fmt.Println("---- delete start ----")
-	deleteErrs := operation.DeleteObjects(config.Bucket)
-	utils.LogErrors(deleteErrs)
-	fmt.Println("---- delete end ----")
+	var incremental *operation.IncrementalConfig
+	if config.IsIncremental {
+		fmt.Println("---- [incremental] ---->")
+		incremental, _ = operation.GetRemoteIncrementalConfig(config.Bucket)
+		fmt.Println("<---- [incremental end] ----")
+		fmt.Println()
+	}
+	if !config.IsIncremental || incremental == nil {
+		// TODO: delete after upload
+		fmt.Println("---- [delete] ---->")
+		deleteErrs := operation.DeleteObjects(config.Bucket)
+		utils.LogErrors(deleteErrs)
+		fmt.Println("<---- [delete end] ----")
+		fmt.Println()
+	}
 
 	records := utils.WalkDir(config.Folder)
 
-	fmt.Println("---- upload start ----")
-	_, uploadErrs := operation.UploadObjects(config.Folder, config.Bucket, records)
+	fmt.Println("---- [upload] ---->")
+	uploaded, uploadErrs := operation.UploadObjects(config.Folder, config.Bucket, records, incremental)
 	utils.LogErrors(uploadErrs)
-	fmt.Println("---- upload end ----")
+	fmt.Println("<---- [upload end] ----")
+	fmt.Println()
+
+	if config.IsIncremental && incremental != nil {
+		fmt.Println("---- [delete] ---->")
+		deleteErrs := operation.DeleteObjectsIncremental(config.Bucket, incremental)
+		utils.LogErrors(deleteErrs)
+		fmt.Println("<---- [delete end] ----")
+		fmt.Println()
+	}
+
+	if config.IsIncremental {
+		fmt.Println("---- [incremental] ---->")
+		operation.UploadIncrementalConfig(config.Bucket, uploaded)
+		fmt.Println("<---- [incremental end] ----")
+		fmt.Println()
+	}
 
 	if len(uploadErrs) > 0 {
 		os.Exit(1)