feat: handle private buckets (#43)

* chore: add new bucket to example site

* feat: use AWS pre-signed URL to get objects from private buckets

* test: add e2e test to check that the example site has 1502 images

* chore: remove region plugin option

The option is not required anymore when using pre-signed URLs

* revert: "chore: remove region plugin option"

This reverts commit d870d3c.

The AWS SDK apparently still expects a region.

* chore(scripts): add local start script with additional env cleanup

* docs: add READMEs to the starter and testing directories

* docs(tests): add details to the testing README

* fix: fix testing folder structure

Cypress sees any file in the integrationFolder as a spec. The spec had to be nexted one level deeper to allow for the README

* docs: update README

* feat: get signed URL only for image files in onCreateNode

BREAKING CHANGE: the Url key doesn't exist on s3Object anymore. There is not a public URL for private buckets and file nodes are now sourced with pre-signed URLs that shouldn't be added to the schema.

Author: robinmetral

README.md

@@ -2,7 +2,11 @@
 
 A Gatsby plugin to source objects and images from AWS S3.
 
-## Install
+## Getting started
+
+### Gatsby setup
+
+Install the plugin:
 
 ```bash
 # with npm
@@ -11,18 +15,13 @@ npm install @robinmetral/gatsby-source-s3
 yarn add @robinmetral/gatsby-source-s3
 ```
 
-## Configure
-
-Declare the plugin in your `gatsby-config.js`, taking care to pass your AWS
-credentials as
+Declare it in your `gatsby-config.js`, making sure to pass your AWS credentials
+as
 [environment variables](https://www.gatsbyjs.org/docs/environment-variables/):
 
 ```javascript
-// configure dotenv
-// see https://www.gatsbyjs.org/docs/environment-variables
-require("dotenv").config({
-  path: `.env.${process.env.NODE_ENV}`
-});
+// gatsby-config.js
+require("dotenv").config();
 
 module.exports = {
   plugins: [
@@ -32,63 +31,70 @@ module.exports = {
         aws: {
           accessKeyId: process.env.AWS_ACCESS_KEY_ID,
           secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
-          region: process.env.AWS_REGION
+          region: process.env.AWS_REGION,
         },
-        buckets: ["my-bucket", "my-second-bucket"]
-      }
-    }
-  ]
+        buckets: ["my-bucket", "my-second-bucket"],
+      },
+    },
+  ],
 };
 ```
 
-Currently, your buckets will need to be configured for public access with this
-access policy: (add your bucket name under `Statement.Resource`)
+### AWS setup
+
+You can use the plugin both with private and public buckets.
 
 ```json
-{
-  "Version": "2008-10-17",
-  "Statement": [
-    {
-      "Sid": "AllowPublicRead",
-      "Effect": "Allow",
-      "Principal": {
-        "AWS": "*"
-      },
-      "Action": "s3:GetObject",
-      "Resource": "arn:aws:s3:::my-bucket/*"
-    }
-  ]
-}
+
 ```
 
-## Query
+## Querying S3 objects
 
-S3 objects can be queried in GraphQL as "s3Object" of "allS3Object":
+S3 objects can be queried in GraphQL as "s3Object" or "allS3Object":
 
 ```graphql
 query AllObjectsQuery {
   allS3Object {
     nodes {
-      Key
-      Url
+      Key # the object's key, i.e. file name
+      Bucket # the object's bucket name on S3
+      LastModified # the date the object was last modified
+      Size # the object's size in bytes
+      localFile # the local file node for image objects processed with sharp (see below)
     }
   }
 }
 ```
 
-## Image processing
+### Processing images with sharp
 
 Any images in your bucket(s) will be downloaded by the plugin and stored as
-local files, to be processed with `gatsby-plugin-sharp` and
+local file nodes, to be processed with `gatsby-plugin-sharp` and
 `gatsby-transformer-sharp`.
 
+If you don't have them yet, you will need to add the sharp plugin and
+transformer to your Gatsby site:
+
 ```bash
 # with npm
 npm install gatsby-plugin-sharp gatsby-transformer-sharp
 # with yarn
 yarn add gatsby-plugin-sharp gatsby-transformer-sharp
 ```
 
+```javascript
+// gatsby-config.js
+module.exports = {
+  plugins: [
+    // ...
+    `gatsby-plugin-sharp`,
+    `gatsby-transformer-sharp`,
+  ],
+};
+```
+
+You can then query the processed images with GraphQL:
+
 ```graphql
 query AllImagesQuery {
   images: allS3Object {
@@ -106,9 +112,19 @@ query AllImagesQuery {
 }
 ```
 
+And use them with `gatsby-image`:
+
+```javascript
+import Img from "gatsby-image";
+
+const Image = ({ s3Object }) => (
+  <Img fluid={s3Object.localFile.childImageSharp.fluid} />
+);
+```
+
 ## Thanks
 
-This plugin was based on Dustin Schau's
+This plugin was initially based on Dustin Schau's
 [`gatsby-source-s3`](https://github.com/DSchau/gatsby-source-s3/) and influenced
-by Jesse Stuart's Typescript
+by Jesse Stuart's TypeScript
 [`gatsby-source-s3-image`](https://github.com/jessestuart/gatsby-source-s3-image).

cypress.json

@@ -1,5 +1,5 @@
 {
-  "integrationFolder": "tests",
+  "integrationFolder": "tests/e2e",
   "screenshotsFolder": "tests/screenshots",
   "fixturesFolder": false,
   "supportFile": false,

examples/gatsby-starter-source-s3/README.md

@@ -0,0 +1,79 @@
+# gatsby-starter-source-s3
+
+This starter is an example of how to source objects from AWS S3 in a Gatsby site
+at build time, using `@robinmetral/gatsby-source-s3`.
+
+It uses a local version of the plugin located in `/src`, and it can be used for
+local development and testing.
+
+To run it locally, you'll need to add the following environment variables in a
+`.env` file:
+
+```bash
+AWS_ACCESS_KEY_ID=""
+AWS_SECRET_ACCESS_KEY=""
+AWS_REGION=""
+```
+
+## AWS S3 setup
+
+This site sources images from three separate buckets:
+
+1. gatsby-source-s3-example (public)
+2. gatsby-source-s3-example-2 (public)
+3. gatsby-source-s3-continuation-token (private)
+
+The first two buckets are set up for public access with the following policy:
+
+```json
+{
+  "Version": "2008-10-17",
+  "Statement": [
+    {
+      "Sid": "AllowPublicRead",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "*"
+      },
+      "Action": "s3:GetObject",
+      "Resource": "arn:aws:s3:::gatsby-source-s3-example/*"
+    }
+  ]
+}
+```
+
+_Note: the resource is the bucket's arn with the `/*` scope._
+
+The third bucket is private, its policy is the default for S3 (i.e. nothing was
+changed when creating the bucket).
+
+## AWS IAM setup
+
+The AWS access keys used by this example are for a `gatsby-source-s3` user to
+which I attached the following access policy:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:ListBucket"],
+      "Resource": [
+        "arn:aws:s3:::gatsby-source-s3-example",
+        "arn:aws:s3:::gatsby-source-s3-example-2",
+        "arn:aws:s3:::gatsby-source-s3-continuation-token"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetObject"],
+      "Resource": [
+        "arn:aws:s3:::gatsby-source-s3-example/*",
+        "arn:aws:s3:::gatsby-source-s3-example-2/*",
+        "arn:aws:s3:::gatsby-source-s3-continuation-token/*"
+      ]
+    }
+  ]
+}
+```

examples/gatsby-starter-source-s3/gatsby-config.js

@@ -2,7 +2,7 @@ require("dotenv").config();
 
 module.exports = {
   siteMetadata: {
-    title: `gatsby-starter-source-s3`
+    title: `gatsby-starter-source-s3`,
   },
   plugins: [
     {
@@ -11,13 +11,17 @@ module.exports = {
         aws: {
           accessKeyId: process.env.AWS_ACCESS_KEY_ID,
           secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
-          region: process.env.AWS_REGION
+          region: process.env.AWS_REGION,
         },
-        buckets: ["gatsby-source-s3-example", "gatsby-source-s3-example-2"]
-      }
+        buckets: [
+          "gatsby-source-s3-example",
+          "gatsby-source-s3-example-2",
+          "gatsby-source-s3-continuation-token",
+        ],
+      },
     },
     // the sharp transformer and plugin are required to process images
     `gatsby-transformer-sharp`,
-    `gatsby-plugin-sharp`
-  ]
+    `gatsby-plugin-sharp`,
+  ],
 };

examples/gatsby-starter-source-s3/src/pages/index.js

@@ -3,12 +3,26 @@ import { graphql } from "gatsby";
 import Img from "gatsby-image";
 
 const IndexPage = ({ data }) => (
-  <>
+  <main style={{ fontFamily: "monospace" }}>
     <h1>{data.site.siteMetadata.title}</h1>
-    {data.allS3Object.nodes.map(image => (
-      <Img fixed={image.localFile.childImageSharp.fixed} alt={image.Key} />
-    ))}
-  </>
+    <div
+      style={{
+        display: "grid",
+        gridTemplateColumns: "repeat(auto-fit,minmax(256px, 1fr))",
+      }}
+      className="images-grid"
+    >
+      {data.allS3Object.nodes.map((image) => (
+        <div className={`s3-image ${image.Key}-${image.Bucket}`}>
+          <Img fixed={image.localFile.childImageSharp.fixed} alt={image.Key} />
+          <br />
+          Key: {image.Key}
+          <br />
+          Bucket: {image.Bucket}
+        </div>
+      ))}
+    </div>
+  </main>
 );
 
 export const IMAGES_QUERY = graphql`
@@ -21,6 +35,7 @@ export const IMAGES_QUERY = graphql`
     allS3Object {
       nodes {
         Key
+        Bucket
         localFile {
           childImageSharp {
             fixed(width: 256) {

package.json

@@ -34,8 +34,9 @@
   "scripts": {
     "build": "tsc",
     "lint": "eslint '*/**/*.{ts,tsx}'",
-    "prestart": "yarn build && npm pack && (cd examples/gatsby-starter-source-s3 && yarn install && yarn add file:../../robinmetral-gatsby-source-s3-0.0.0-semantically-released.tgz)",
+    "prestart": "yarn build && npm pack && (cd examples/gatsby-starter-source-s3 && yarn install)",
     "start": "(cd examples/gatsby-starter-source-s3 && gatsby build && gatsby serve)",
+    "start:local": "yarn cache clean && (cd examples/gatsby-starter-source-s3 && rm -rf node_modules .cache public yarn.lock) && yarn start",
     "test": "cypress run",
     "e2e": "start-server-and-test http://localhost:9000"
   },

src/gatsby-node.ts

@@ -1,6 +1,8 @@
 import { createRemoteFileNode } from "gatsby-source-filesystem";
 import AWS = require("aws-sdk");
 
+const s3 = new AWS.S3();
+
 const isImage = (key: string): boolean =>
   /\.(jpe?g|png|webp|tiff?)$/i.test(key);
 
@@ -26,8 +28,6 @@ export async function sourceNodes(
   AWS.config.update(awsConfig);
 
   // get objects
-  const s3 = new AWS.S3();
-
   const getS3ListObjects = async (params: {
     Bucket: string;
     ContinuationToken?: string;
@@ -86,19 +86,11 @@ export async function sourceNodes(
     const objects = allBucketsObjects.reduce((acc, val) => acc.concat(val), []);
 
     // create file nodes
-    // todo touch nodes if they exist already
     objects?.forEach(async (object) => {
-      const { Key, Bucket } = object;
-      const { region } = awsConfig;
-
       createNode({
         ...object,
-        // construct url
-        Url: `https://s3.${
-          region ? `${region}.` : ""
-        }amazonaws.com/${Bucket}/${Key}`,
         // node meta
-        id: createNodeId(`s3-object-${Key}`),
+        id: createNodeId(`s3-object-${object.Key}`),
         parent: null,
         children: [],
         internal: {
@@ -123,9 +115,16 @@ export async function onCreateNode({
 }) {
   if (node.internal.type === "S3Object" && node.Key && isImage(node.Key)) {
     try {
+      // get pre-signed URL
+      const url = s3.getSignedUrl("getObject", {
+        Bucket: node.Bucket,
+        Key: node.Key,
+        Expires: 60,
+      });
+
       // download image file and save as node
       const imageFile = await createRemoteFileNode({
-        url: node.Url,
+        url,
         parentNodeId: node.id,
         store,
         cache,