Skip to content

Commit 9b0307d

Browse files
author
VIJAY UNIYAL
committed
formatting
1 parent b585a39 commit 9b0307d

File tree

4 files changed

+149
-142
lines changed

4 files changed

+149
-142
lines changed

‎sitemap/generator.js

+57-53
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,31 @@ const yargs = require('yargs');
33
const fs = require('fs');
44
const linksProcess = require('./sitemapIndex');
55
const axios = require('axios');
6-
const axiosRetry = require('axios-retry');
76
const https = require('https');
87

98

109
const argv = yargs
11-
.option('api', {
12-
alias: 'a',
13-
description: 'Base URL for the catalog search api, e.g. http://localhost/onestop-search',
14-
type: 'string',
15-
required: true
16-
})
17-
.option('website', {
18-
alias: 'w',
19-
description: 'Base URL for the catalog web page, e.g. http://localhost/onestop',
20-
type: 'string',
21-
required: true
22-
})
23-
.option('pageSize', {
24-
alias: 'n',
25-
description: 'Number of collections to request at a time',
26-
type: 'number',
27-
default: 10
28-
})
29-
.help()
30-
.alias('help', 'h')
31-
.argv;
10+
.option('api', {
11+
alias: 'a',
12+
description: 'Base URL for the catalog search api, e.g. http://localhost/onestop-search',
13+
type: 'string',
14+
required: true
15+
})
16+
.option('website', {
17+
alias: 'w',
18+
description: 'Base URL for the catalog web page, e.g. http://localhost/onestop',
19+
type: 'string',
20+
required: true
21+
})
22+
.option('pageSize', {
23+
alias: 'n',
24+
description: 'Number of collections to request at a time',
25+
type: 'number',
26+
default: 10
27+
})
28+
.help()
29+
.alias('help', 'h')
30+
.argv;
3231

3332

3433
//TODO - When pulling from cedar devs update API URL new path
@@ -48,46 +47,51 @@ let options = {
4847
},
4948
data: {
5049
"summary": false,
51-
"sort": [{"stagedDate": "asc"}],
50+
"sort": [{
51+
"stagedDate": "asc"
52+
}],
5253
"search_after": [0],
5354
"queries": [],
54-
"page": {"max": pageSize}
55+
"page": {
56+
"max": pageSize
57+
}
5558
}
5659
};
5760

5861
//recursively pages API
59-
let pageApi = async function(options, collectionList){
62+
let pageApi = async function (options, collectionList) {
6063
console.log("Requesting page for data after: " + options.data.search_after[0]);
6164
await axios(options)
62-
.then((response) => {
63-
console.log("Response status: " + response.status);
64-
if (response.status == 200) {
65-
let body = response.data;
66-
//If we got data back, process it and then keep going until we dont have anymore data
67-
// TODO catch body.error
68-
if (body && body.data.length > 0) {
69-
//grab the last staged date, we will need it for the subsequent request
70-
const lastStagedDate = body.data[body.data.length - 1].attributes.stagedDate;
71-
//update the options
72-
options.data.search_after = [lastStagedDate];
73-
//create the data structure we need for the sitemap tool
74-
var bodyDataObjectList = processBodyData(body);
75-
//add it to the list
76-
collectionList = [...collectionList, ...bodyDataObjectList];
77-
console.log("Received " + body.data.length + " items, continue paging...");
78-
//get the next page
79-
collectionList = pageApi(options, collectionList)
80-
} else {
81-
console.log("No more data. Generating sitemap...");
82-
}
65+
.then((response) => {
66+
console.log("Response status: " + response.status);
67+
if (response.status == 200) {
68+
let body = response.data;
69+
//If we got data back, process it and then keep going until we dont have anymore data
70+
// TODO catch body.error. Check if response status is not 200
71+
if (body && body.data.length > 0) {
72+
//grab the last staged date, we will need it for the subsequent request
73+
const lastStagedDate = body.data[body.data.length - 1].attributes.stagedDate;
74+
//update the options
75+
options.data.search_after = [lastStagedDate];
76+
//create the data structure we need for the sitemap tool
77+
var bodyDataObjectList = processBodyData(body);
78+
//add it to the list
79+
collectionList = [...collectionList, ...bodyDataObjectList];
80+
console.log("Received " + body.data.length + " items, continue paging...");
81+
//get the next page
82+
collectionList = pageApi(options, collectionList)
83+
} else {
84+
console.log("No more data. Generating sitemap...");
8385
}
84-
})
85-
.catch(function (error) {
86-
console.log("ERROR");
87-
console.log(error);
88-
});
89-
return collectionList;
86+
}
87+
})
88+
//TODO - Future story more error handling
89+
.catch(function (error) {
90+
console.log("ERROR");
91+
console.log(error);
92+
});
93+
return collectionList;
9094
}
9195

9296
//page the api, create sitemap
93-
pageApi(options, []).then((listOfLinks) => linksProcess(listOfLinks));
97+
pageApi(options, []).then((listOfLinks) => linksProcess(listOfLinks));

‎sitemap/sitemapIndex.js

+9-22
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const { createReadStream, createWriteStream } = require('fs');
1+
const { createReadStream, createWriteStream} = require('fs');
22
const { resolve } = require('path');
33
const { createGzip } = require('zlib')
44
const { Readable } = require('stream')
@@ -17,13 +17,15 @@ const linksProcess = (links) => {
1717
module.exports = linksProcess;
1818

1919
const sms = new SitemapAndIndexStream({
20-
limit: 2, // defaults to 45k
20+
limit: 50, // defaults to 45k
2121
// SitemapAndIndexStream will call this user provided function every time
2222
// it needs to create a new sitemap file. You merely need to return a stream
2323
// for it to write the sitemap urls to and the expected url where that sitemap will be hosted
2424
hostname: 'https://cedardevs.org',
2525
getSitemapStream: (i) => {
26-
const sitemapStream = new SitemapStream({ hostname: 'https://cedardevs.org' });
26+
const sitemapStream = new SitemapStream({
27+
hostname: 'https://cedardevs.org'
28+
});
2729
const path = `./sitemap-${i}.xml`;
2830

2931
sitemapStream
@@ -36,10 +38,10 @@ const sms = new SitemapAndIndexStream({
3638

3739
// or reading straight from an in-memory array
3840
sms
39-
.pipe(createGzip())
40-
.pipe(createWriteStream(resolve('./sitemap-index.xml.gz')));
41+
.pipe(createGzip())
42+
.pipe(createWriteStream(resolve('./sitemap-index.xml.gz')));
4143

42-
/*
44+
/* Format passed into the sitemap library to generate sitemap files
4345
const exampleList = [{
4446
url: 'http://localhost/onestop/collections/details/0561ce74-bc07-4dd4-bf22-8c73befe9497',
4547
changefreq: 'daily',
@@ -50,24 +52,9 @@ const exampleList = [{
5052
changefreq: 'daily',
5153
lastmod: '2021-01-21T22:59:57.591Z'
5254
},
53-
{
54-
url: 'http://localhost/onestop/collections/details/dac3dbdf-fe19-4b86-aeb1-2fef83a26f51',
55-
changefreq: 'daily',
56-
lastmod: '2021-01-21T22:59:57.652Z'
57-
},
58-
{
59-
url: 'http://localhost/onestop/collections/details/41633d5a-d8b3-482a-92ff-8ca6833c63d8',
60-
changefreq: 'daily',
61-
lastmod: '2021-01-21T22:59:57.719Z'
62-
},
63-
{
64-
url: 'http://localhost/onestop/collections/details/150f59f9-7827-4eca-9156-6ae5c1640308',
65-
changefreq: 'daily',
66-
lastmod: '2021-01-21T22:59:57.779Z'
67-
},
6855
{
6956
url: 'http://localhost/onestop/collections/details/62e7d363-0540-40ed-b89e-4a5e33a28ff2',
7057
changefreq: 'daily',
7158
lastmod: '2021-01-21T22:59:57.848Z'
7259
}];
73-
*/
60+
*/

‎sitemap/transformUtils.js

+12-11
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ const convertCollectionToXml = (baseUrl, collection) => {
55
var stagedDate = collection.attributes.stagedDate;
66
var formattedDate = Unix_TimeStamp(stagedDate);
77

8-
return`
8+
return `
99
<url>
1010
<loc>${baseUrl}/onestop/collections/details/${collection.id}</loc>
1111
<lastmod>${formattedDate}</lastmod>
@@ -18,15 +18,17 @@ const convertCollectionToObject = (baseUrl, collection) => {
1818

1919
var stagedDate = collection.attributes.stagedDate;
2020
var formattedDate = Unix_TimeStamp(stagedDate);
21-
var collObject = { url: `${baseUrl}/collections/details/${collection.id}`,
22-
changefreq: `daily`,
23-
lastmod: `${formattedDate}`};
21+
var collObject = {
22+
url: `${baseUrl}/collections/details/${collection.id}`,
23+
changefreq: `daily`,
24+
lastmod: `${formattedDate}`
25+
};
2426

2527
return collObject;
2628
};
2729

2830
//Helper method for convertCollectiontoXML, transforms the collection's stagedDate to W3 DateTime format.
29-
const Unix_TimeStamp = (t) =>{
31+
const Unix_TimeStamp = (t) => {
3032

3133
var dt = new Date(t)
3234
var n = dt.toISOString();
@@ -43,16 +45,15 @@ const processBodyData = (body) => {
4345
var i = 0;
4446

4547
//Truthy check if(body), will evaluate to true if value is not null,undefined, NaN, empty string, 0, or false
46-
if ( body ) {
48+
if (body) {
4749
body.data.forEach((d) => {
48-
//console.log(convertCollectionToObject(webBase, d));
4950
bodyDataObject[i++] = convertCollectionToObject(webBase, d);
50-
});
51+
});
5152
} else {
5253
console.log("processBodyData body check error");
5354
}
54-
55-
return bodyDataObject;
55+
56+
return bodyDataObject;
5657
}
5758

58-
module.exports = processBodyData;
59+
module.exports = processBodyData;

0 commit comments

Comments
 (0)