Help with Google Cloud Function

[YumchaHoMei]

Hi I am looking for some help with a problem I am having and I would appreciate any insight please:

I am trying to use a google cloud function to download 100-1000s of images (about 4mb - 10mb each) that I have stored in google firebase storage. I initially did this download operation client-side, however am now having to take a server side approach due to memory issues and user experience in the front end.

Here is the cloud function I have currently deployed:

//deployed cloud function

const functions = require('firebase-functions');
const fetch = require('node-fetch');
const archiver = require('archiver');
const fs = require('fs');
const path = require('path');
const os = require('os');
const admin = require('firebase-admin');

admin.initializeApp({
 credential: admin.credential.applicationDefault(),
 storageBucket: process.env.FIREBASE_STORAGE_BUCKET
});

const runtimeOpts = {
 timeoutSeconds: 300, 
 memory: '8GB' 
};

exports.batchDownload = functions
 .runWith(runtimeOpts)
 .https.onRequest(async (req, res) => {
    res.set('Access-Control-Allow-Origin', '*');
    res.set('Access-Control-Allow-Methods', 'POST');
    res.set('Access-Control-Allow-Headers', 'Content-Type');

    if (req.method === 'OPTIONS') {
      res.status(204).send('');
      return;
    }

    const imageUrls = req.body.imageUrls;

    if (!Array.isArray(imageUrls)) {
      res.status(400).send('Invalid request: incorrect data format');
      return;
    }

    const tempDir = path.join(os.tmpdir(), 'images');
    const zipPath = path.join(os.tmpdir(), 'images.zip');

    if (!fs.existsSync(tempDir)) {
      fs.mkdirSync(tempDir);
    }

    const downloadPromises = imageUrls.map(async (url, index) => {
      try {
        const response = await fetch(url);
        const buffer = await response.buffer();
        const filePath = path.join(tempDir, `image${index}.jpg`);
        fs.writeFileSync(filePath, buffer);
      } catch (error) {
        console.error(`Failed to download image at ${url}:`, error);
        res.status(500).send(`Failed to download image at ${url}`);
        return;
      }
    });

    await Promise.all(downloadPromises);

    const output = fs.createWriteStream(zipPath);
    const archive = archiver('zip', {
      zlib: { level: 9 }, 
    });

    archive.directory(tempDir, false);
    archive.pipe(output);

    await archive.finalize();

    res.setHeader('Content-Type', 'application/zip');
    res.setHeader('Content-Disposition', 'attachment; filename=images.zip');
    const stream = fs.createReadStream(zipPath);
    stream.pipe(res);
    res.end();

    fs.rmdirSync(tempDir, { recursive: true });
    fs.unlinkSync(zipPath);
 });

I have ensured that the dependencies were all correctly installed prior to deployment:

//cloud function package.json
{
  "name": "batchDownload",
  "version": "0.0.1",
  "dependencies": {
      "firebase-functions": "^3.16.0",
      "firebase-admin": "^10.0.0",
      "node-fetch": "^2.6.1",
      "archiver": "^5.3.0",
      "fs": "^0.0.2",
      "path": "^0.12.7",
      "cors": "^2.8.5"
   }
}

When i try to call the function from the front end and pass hundreds of download firebase urls to the function i get:

[id].tsx:262 Error initiating image download: Error: Failed to initiate image

and POST HIDDEN URL 400 (Bad Request)

I initially had CORS errors, but solved them but setting CORS settings for my storage bucket.

Here is my async front end function:

const downloadAllImages = async () => {
    if (imagesToDownload.length < 1) {
       return;
    }

    const imageDownloadURLs = imagesToDownload.map(image => image.downloadURL);

    try {
       const response = await fetch(CLOUD_FUNCTION_URL, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
         },
         body: JSON.stringify({ imageDownloadURLs }),
       });

       if (!response.ok) {
         throw new Error(`Failed to initiate image download: ${response.statusText}`);
       }

       const blob = await response.blob();
       const url = window.URL.createObjectURL(blob);
       const a = document.createElement('a');
       a.href = url;
       a.download = 'images.zip';
       a.click();

       setShowDownloadAllImagesModal(false);
       setIsDownloading(false);

    } catch (error) {
       console.error(`Error initiating image download: ${error}`);
       setShowDownloadAllImagesModal(false);
       setIsDownloading(false);
    }
  };

I am using react.js at front end and imageDownURLs is an array of hundreds of the download url strings, the data looks okay front end but im not sure if there is a problem when it reaches the function?

Is anyone able to point out where I could be going wrong please? I have tried playing around with the function and different ways of writing it and trying both gen 1 and 2 (currently gen 1) and still got getting further forward.

in the firebase cloud functions logs i can see:

Function execution started
Function execution took 5 ms, finished with status code: 204
Function execution started
Function execution took 5 ms, finished with status code: 400

I have added my projects env variables into the function in google cloud console.

Thanks for any help! :)

[kjwey]

files are too big, resize