Firebase - Server Error the specified payload is too large

23.07.2021 — nodejs, firebase, time-wasted — 1 min read

Initially, When i started to have a look at this, the storage was at 2.2Gb. You can see that in the Before Storage Graph in the left-bottom image. That seemed to be high, i know why and which node in the firebase RTDB contributed to it. So basically my idea is to download all the data in that down and cleared it down.

Failed attempts are

Export options. Failed to work as from a single location you cannot export data exceeding 256 MB.
V1 - Since i had the keys, programmatically tried to retreive data via NodeJS and approach used is and got the below error.
- Using .every() loop method and pushing promises to array
- Using bluebird Promise.map to process the promises with concurrency(2)

1[2021-07-19T17:25:02.560Z]  @firebase/database: FIREBASE INTERNAL ERROR: Server Error: The specified payload is too large, please request a location with less data.

Successful approach that worked is,

Created an outer loop using .slice() so that rest of the program processes around 10 chunks
Using .every() loop method and pushing promises to array
Using bluebird Promise.map to process the promises with concurrency 2

Below is the code,

1const firebase = require('firebase-admin');
2const fs = require("fs")
3var Promise = require('bluebird');
4
5var initializeFirebase = () => {
6    var serviceAccount = require("./private/SA-firebase-adminsdk.json");
7    firebase.initializeApp({
8        credential: firebase.credential.cert(serviceAccount),
9        databaseURL: "https://<<fdb>>.firebaseio.com"  
10    });
11};
12
13//Functions
14const writeFile = (filename, data) =>{
15    return new Promise(function(resolve, reject) {
16        fs.writeFile(filename, JSON.stringify(data, null, 4), function(err) {
17            if (err) reject(err);
18            else resolve("Written filename "+filename);
19        });
20    });            
21};
22
23var readFB = (fbPath, scCode) =>{
24    return new Promise( (resolve, reject) => {
25        return firebase.database().ref().child(fbPath).once('value').then(function(snapshot) {
26            // console.log(snapshot.val());
27            var obj = {[scCode] : snapshot.val()};
28            resolve(obj) ;
29        })        
30    });
31}
32
33 var start = async () =>{
34
35    if (!firebase.apps.length) {
36        initializeFirebase();
37    }
38
39    var codes = fs.readFileSync('./data/code_list.txt', 'utf8').replace(/\r/g, '') .split('\n');
40    console.log('Total sc_code list = '+codes.length);
41    
42    var promises = [];
43    var i,j,temp_arr, chunk = 10;
44    for(i=0, j=codes.length;i<j;i+=chunk){
45        temp_arr = codes.slice(i, i+chunk);
46
47        await temp_arr.every(async code => {        
48            console.log(code);
49            let fbPath = 'BTD_AnnouncementsByCode/'+code;
50            await promises.push(readFB(fbPath, code))
51        });
52    
53        var outPath = './data/Announcements/'
54
55        //Using Promise.map to control concurrency
56        await Promise.map(promises, async (result)=>{
57            let [key, value] = Object.entries(result)[0]; 
58            // console.log(value);
59            if(!value){
60                console.log('Null value '+key);
61                return;
62            }
63            let [key2, value2] = Object.entries(value)[0]; 
64            console.log('Key:'+key+' ; Value:'+value2.SLONGNAME);
65
66            let fName = key+' - '+value2.SLONGNAME.replace(/\.+$/, "")+'.json';
67            await writeFile(outPath+fName, value).then( response =>{
68                console.log(response);
69            }).catch( err => {
70                console.log('ERROR : START : '+err);    
71                if (firebase.apps.length) { firebase.app().delete(); }
72            });   
73        }, {concurrency: 10});
74
75    }
76            
77    if (firebase.apps.length) { firebase.app().delete(); }
78
79    process.exit();  //Added this as NodeJS application doesn't exit properly
80        
81};
82
83start();

After doing this, still there were more data and keys of those i didn't have and this is where Firebase Database REST API came to help.

1curl "https://btd-in2.firebaseio.com/BTD_AnnouncementsByCode.json?shallow=true&print=pretty" > code_list4.txt

By executing this command, i got just the keys without the data underlying it, thats the shallow trick.

shallow : This is an advanced feature, designed to help you work with large datasets without needing to download everything. Set this to true to limit the depth of the data returned at a location. If the data at the location is a JSON primitive (string, number or boolean), its value will simply be returned. If the data snapshot at the location is a JSON object, the values for each key will be truncated to true.

Plan for future clean-ups is to incorporate the firebase rest api into the above program like,

Execute the Firebase REST API with Shallow via nodejs and retreive the keys
Create an outer loop using .slice() so that rest of the program processes around 10 chunks
Using .every() loop method and pushing promises to array
Using bluebird Promise.map to process the promises with concurrency 2

Why in the first place ended up in this activity

Assumption, i will be able to retreive using program or export via firebase feature
Didn't plan for archieving or decommissioning while creating it initially.

Thank you for reading