feat: 5839 Add file attachment support for ProjectHub payloads

frontend/package.json

@@ -27,6 +27,7 @@
     "codemirror": "^5.65.0",
     "cronstrue": "^2.59.0",
     "file-saver": "^2.0.5",
+    "file-type": "^21.3.0",
     "idb": "8.0.3",
     "js-yaml": "^4.1.0",
     "leader-line": "^1.0.7",

frontend/src/app/modules/policy-engine/policy-viewer/blocks/action-block/action-block.component.ts

@@ -6,6 +6,8 @@ import { PolicyHelper } from 'src/app/services/policy-helper.service';
 import { WebSocketService } from 'src/app/services/web-socket.service';
 import { RegisteredService } from '../../../services/registered.service';
 import { DynamicMsalAuthService } from '../../../services/dynamic-msal-auth.service';
+import { IPFSService } from 'src/app/services/ipfs.service';
+import { BlockType } from '@guardian/interfaces';
 
 /**
  * Component for display block of 'requestVcDocument' type.
@@ -19,6 +21,7 @@ export class ActionBlockComponent implements OnInit {
     @Input('id') id!: string;
     @Input('policyId') policyId!: string;
     @Input('static') static!: any;
+    @Input('dryRun') dryRun!: any;
 
     loading: boolean = true;
     socket: any;
@@ -43,7 +46,8 @@ export class ActionBlockComponent implements OnInit {
         private wsService: WebSocketService,
         private policyHelper: PolicyHelper,
         private toastr: ToastrService,
-        private dynamicMsalAuthService: DynamicMsalAuthService
+        private dynamicMsalAuthService: DynamicMsalAuthService,
+        private ipfsService: IPFSService,
     ) {
     }
 
@@ -142,7 +146,12 @@ export class ActionBlockComponent implements OnInit {
     private createInstance(config: any) {
         const code: any = this.registeredService.getCode(config.blockType);
         if (code) {
-            return new code(config, this.policyEngineService, this.dynamicMsalAuthService, this.toastr);
+            if (config.blockType === BlockType.IpfsTransformationUIAddon) {
+                return new code(config, this.ipfsService, this.dryRun);
+            }
+            else{
+                return new code(config, this.policyEngineService, this.dynamicMsalAuthService, this.toastr);
+            }
         }
         return null;
     }

frontend/src/app/modules/policy-engine/policy-viewer/code/ipfs-transformation-ui-addon.ts

@@ -0,0 +1,230 @@
+import { IPFSService } from "src/app/services/ipfs.service";
+import { firstValueFrom } from 'rxjs';
+import { fileTypeFromBuffer } from 'file-type';
+
+interface DocumentData {
+    document: any;
+    params: any;
+    history: any[];
+}
+
+interface IpfsMatch {
+    fullMatch: string;
+    cid: string;
+}
+
+enum TransformationIpfsLinkType {
+    Base64 = 'base64',
+    IpfsGateway = 'ipfsGateway'
+}
+
+export class IpfsTransformationUIAddonCode {
+    private readonly ipfsPattern: RegExp = /ipfs:\/\/([a-zA-Z0-9]+)/;
+    private cache: Map<string, string> = new Map();
+
+    private readonly transformationType: string;
+    private readonly ipfsGatewayTemplate: string;
+
+    constructor(
+        private config: any,
+        private ipfsService: IPFSService,
+        private dryRun: boolean
+    ) {
+        this.transformationType = this.config.transformationType;
+        this.ipfsGatewayTemplate = this.config.ipfsGatewayTemplate;
+    }
+
+    public async run(data: DocumentData): Promise<DocumentData> {
+        try {
+            await this.processDocument(data.document);
+            return data;
+        } catch (error) {
+            console.error('Error processing IPFS transformations:', error);
+            throw error;
+        }
+    }
+
+    private async processDocument(rootDocument: any): Promise<void> {
+        if (!rootDocument || typeof(rootDocument) !== 'object') {
+            return;
+        }
+
+        const stack: any[] = [rootDocument];
+        const tasks: Promise<void | any>[] = [];
+
+        while (stack.length) {
+            const documentObject = stack.pop();
+            if (Array.isArray(documentObject)) {
+                for (let i = 0; i < documentObject.length; i++) {
+                    const documentValue = documentObject[i];
+                    if (typeof(documentValue) === 'string' && documentValue.startsWith('ipfs://')) {
+                        tasks.push(this.processIpfsString(documentValue).then(res => { 
+                            documentObject[i] = res; 
+                        }));
+                    } else if (documentValue && typeof(documentValue) === 'object') {
+                        stack.push(documentValue);
+                    }
+                }
+            } else {
+                for (const key in documentObject) {
+                    const value = documentObject[key];
+                    if (typeof(value) === 'string' && value.startsWith('ipfs://')) {
+                        tasks.push(this.processIpfsString(value).then(res => { 
+                            documentObject[key] = res; 
+                        }));
+                    } else if (value && typeof(value) === 'object') {
+                        stack.push(value);
+                    }
+                }
+            }
+        }
+
+        if (tasks.length) {
+            await Promise.all(tasks);
+        }
+    }
+
+
+    private async processIpfsString(ipfsString: string): Promise<any> {
+        const match = this.ipfsPattern.exec(ipfsString);
+        if (!match) {
+            return ipfsString;
+        }
+
+        const cid = match[1];
+
+        if (this.transformationType === TransformationIpfsLinkType.IpfsGateway) {
+            return this.convertToIpfsGateway(cid);
+        } else if (this.transformationType === TransformationIpfsLinkType.Base64) {
+            return await this.convertToBase64({ fullMatch: ipfsString, cid });
+        }
+
+        return ipfsString;
+    }
+
+    private convertToIpfsGateway(cid: string): any {
+        let gatewayUrl = "";
+        if (this.ipfsGatewayTemplate.includes('{cid}')) {
+            gatewayUrl = this.ipfsGatewayTemplate.replace('{cid}', cid);
+        } else {
+            gatewayUrl = `${this.ipfsGatewayTemplate}/${cid}`;
+        }
+        return { resourceUrl: gatewayUrl };
+    }
+
+    private async convertToBase64(match: IpfsMatch): Promise<any> {
+        if (!match.cid) {
+            return match.fullMatch;
+        } 
+
+        if (this.cache.has(match.cid)) {
+            const cachedBase64 = this.cache.get(match.cid)!;
+            return { base64String: cachedBase64 };
+        }
+
+        try {
+            const arrayBuffer = await this.loadFileFromIpfs(match.cid);
+            const dataUrl = await this.arrayBufferToDataUrl(arrayBuffer);
+            this.cache.set(match.cid, dataUrl);
+
+            return { base64String: dataUrl };
+        } catch (error) {
+            console.error(`convertToBase64 by CID ${match.cid}:`, error);
+            return match.fullMatch;
+        }
+    }
+
+    private async loadFileFromIpfs(cid: string): Promise<ArrayBuffer> {
+        const isDryRun = this.dryRun === true;
+        const file$ = isDryRun 
+            ? this.ipfsService.getFileFromDryRunStorage(cid)
+            : this.ipfsService.getFile(cid);
+
+        return await firstValueFrom(file$);
+    }
+
+    private async arrayBufferToDataUrl(buffer: ArrayBuffer): Promise<string> {
+        const base64 = this.arrayBufferToBase64(buffer);
+
+        const fileType = await fileTypeFromBuffer(buffer);
+        let mimeType = 'application/octet-stream'; 
+
+        if (fileType) {
+            mimeType = fileType.mime;
+        } else if (this.isTextBuffer(buffer)) {
+            mimeType = this.isCsvBuffer(buffer) ? 'text/csv' : 'text/plain';
+        }
+
+        return `data:${mimeType};base64,${base64}`;
+    }
+
+    private isTextBuffer(buffer: ArrayBuffer): boolean {
+        const checkSize = Math.min(1024, buffer.byteLength);
+        const chunk = new Uint8Array(buffer, 0, checkSize);
+        for (let i = 0; i < chunk.length; i++) {
+            const byte = chunk[i];
+            if (byte === 0 || (byte < 9 || (byte > 13 && byte < 32)) && byte > 127) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private isCsvBuffer(buffer: ArrayBuffer): boolean {
+        try {
+            const checkSize = Math.min(64 * 1024, buffer.byteLength);
+            const chunk = new Uint8Array(buffer, 0, checkSize);
+
+            // skip UTF-8 BOM
+            let offset = 0;
+            if (chunk[0] === 0xEF && chunk[1] === 0xBB && chunk[2] === 0xBF) {
+                offset = 3;
+            }
+
+            const text = new TextDecoder().decode(chunk);
+            const lines = text.split(/\r?\n/).slice(0, 5);
+
+            const firstLine = lines[0].trim();
+            if (!firstLine) {
+                return false;
+            }
+
+            const delimiters = [',', ';', '\t'];
+
+            for (const delimiter of delimiters) {
+                const delimiterCount = (firstLine.match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+
+                if (delimiterCount > 0) {
+                    const isConsistent = lines.slice(1, 4).every(line => {
+                        const lineTrimmed = line.trim();
+                        if (!lineTrimmed) {
+                            return true;
+                        }
+                        const count = (lineTrimmed.match(new RegExp(`\\${delimiter}`, 'g')) || []).length;
+                        return Math.abs(count - delimiterCount) <= 1;
+                    });
+
+                    if (isConsistent && !text.match(/<\w+>/)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        } catch {
+            return false;
+        }
+    }
+
+    private arrayBufferToBase64(buffer: ArrayBuffer): string {
+        const CHUNK_SIZE_IN_KB = 32 * 1024;
+        const chunks: string[] = [];
+        const bytes = new Uint8Array(buffer);
+
+        for (let i = 0; i < bytes.length; i += CHUNK_SIZE_IN_KB) {
+            const chunk = bytes.subarray(i, i + CHUNK_SIZE_IN_KB);
+            chunks.push(String.fromCharCode(...chunk));
+        }
+
+        return btoa(chunks.join(''));
+    }
+}

frontend/src/app/modules/policy-engine/services/block-icons.ts

@@ -60,5 +60,6 @@ BlockIcons[BlockType.HttpRequestUIAddon] = 'globe';
 BlockIcons[BlockType.TransformationUIAddon] = 'chart-bar';
 BlockIcons[BlockType.GlobalEventsReaderBlock] = 'cloud-download';
 BlockIcons[BlockType.GlobalEventsWriterBlock] = 'cloud-upload';
+BlockIcons[BlockType.IpfsTransformationUIAddon] = 'file-edit';
 
 export default BlockIcons;

frontend/src/app/modules/policy-engine/services/blocks-information.ts

@@ -60,6 +60,7 @@ import { WipeConfigComponent } from '../policy-configuration/blocks/tokens/wipe-
 import { MathConfigComponent } from '../policy-configuration/blocks/calculate/math-config/math-config.component';
 import { GlobalEventsReaderBlockComponent } from '../policy-viewer/blocks/global-events-reader-block/global-events-reader-block.component';
 import { GlobalEventsWriterBlockComponent } from "../policy-viewer/blocks/global-events-writer-block/global-events-writer-block.component";
+import { IpfsTransformationUIAddonCode } from '../policy-viewer/code/ipfs-transformation-ui-addon';
 
 const Container: IBlockSetting = {
     type: BlockType.Container,
@@ -888,6 +889,16 @@ const TransformationUIAddon: IBlockSetting = {
     code: TransformationUIAddonCode,
 }
 
+const IpfsTransformationUIAddon: IBlockSetting = {
+    type: BlockType.IpfsTransformationUIAddon,
+    icon: BlockIcons[BlockType.IpfsTransformationUIAddon],
+    group: BlockGroup.Main,
+    header: BlockHeaders.Addons,
+    factory: null,
+    property: null,
+    code: IpfsTransformationUIAddonCode,
+}
+
 export default [
     Container,
     Step,
@@ -944,5 +955,6 @@ export default [
     TransformationButtonBlock,
     IntegrationButtonBlock,
     HttpRequestUIAddon,
-    TransformationUIAddon
+    TransformationUIAddon,
+    IpfsTransformationUIAddon
 ];

frontend/src/app/modules/policy-engine/themes/default.ts

@@ -93,6 +93,7 @@ export const defaultTheme = {
                 'dataTransformationAddon',
                 'transformationUIAddon',
                 'httpRequestUIAddon',
+                'ipfsTransformationUIAddon',
             ]
         }
     ],

interfaces/src/type/block.type.ts

@@ -60,4 +60,5 @@ export enum BlockType {
     DataTransformationAddon = 'dataTransformationAddon',
     HttpRequestUIAddon = 'httpRequestUIAddon',
     TransformationUIAddon = 'transformationUIAddon',
+    IpfsTransformationUIAddon = 'ipfsTransformationUIAddon',
 }

policy-service/src/policy-engine/block-validators/block-validator.ts

@@ -62,6 +62,7 @@ import { HttpRequestUIAddon } from './blocks/http-request-ui-addon.js';
 import { TransformationUIAddon } from './blocks/transformation-ui-addon.js';
 import {GlobalEventsWriterBlock} from './blocks/global-events-writer-block.js';
 import {GlobalEventsReaderBlock} from './blocks/global-events-reader-block.js';
+import { IpfsTransformationUIAddon } from './blocks/ipfs-transformation-ui-addon.js';
 
 export const validators = [
     InterfaceDocumentActionBlock,
@@ -121,6 +122,7 @@ export const validators = [
     TransformationUIAddon,
     GlobalEventsWriterBlock,
     GlobalEventsReaderBlock,
+    IpfsTransformationUIAddon
 ];
 
 /**

policy-service/src/policy-engine/block-validators/blocks/ipfs-transformation-ui-addon.ts

@@ -0,0 +1,25 @@
+import { BlockValidator, IBlockProp } from '../index.js';
+import { CommonBlock } from './common.js';
+
+/**
+ * IPFS Transformation UI Addon
+ */
+export class IpfsTransformationUIAddon {
+    /**
+     * Block type
+     */
+    public static readonly blockType: string = 'ipfsTransformationUIAddon';
+
+    /**
+     * Validate block options
+     * @param validator
+     * @param config
+     */
+    public static async validate(validator: BlockValidator, ref: IBlockProp): Promise<void> {
+        try {
+            await CommonBlock.validate(validator, ref);
+        } catch (error) {
+            validator.addError(`Unhandled exception ${validator.getErrorMessage(error)}`);
+        }
+    }
+}

policy-service/src/policy-engine/blocks/index.ts

@@ -57,4 +57,5 @@ export { HttpRequestUIAddon } from './http-request-ui-addon.js';
 export { TransformationUIAddon } from './transformation-ui-addon.js';
 export { MathBlock } from './math-block.js';
 export { GlobalEventsWriterBlock } from './global-events-writer-block.js';
+export { IpfsTransformationUIAddon } from './ipfs-transformation-ui-addon.js'
 export { default as GlobalEventsReaderBlock } from './global-events-reader-block.js'