"Unexpected token in JSON at position 0" using JSON.parse in Node with valid JSON

Question

I have been tearing my hair over this hour for several hours.

I have a simple Node server that makes an external API call to receive (massive, e.g. 4+ MB) JSON bits. I use as a template a request that you can get, taken directly from Node docs:

const muniURL = `http://api.511.org/transit/vehiclemonitoring?api_key=${API_KEYS.API_KEY_511}&format=json&agency=sf-muni`;

http.get(muniURL, (res) => {
  const statusCode = res.statusCode;
  const contentType = res.headers['content-type'];
  console.log('Status Code:', statusCode);
  console.log('Content Type:', contentType);

  let error;
  if (statusCode !== 200) {
    error = new Error(`Request Failed.\n` +
                      `Status Code: ${statusCode}`);
  } else if (!/^application\/json/.test(contentType)) {
    error = new Error(`Invalid content-type.\n` +
                      `Expected application/json but received ${contentType}`);
  }
  if (error) {
    console.log(`Request error: ${error.message}`);
    // consume response data to free up memory
    res.resume();
    return;
  }

  res.setEncoding('utf8');
  let rawData = '';
  res.on('data', (chunk) => rawData += chunk);
  res.on('end', () => {
    try {
      const parsedData = JSON.parse(rawData);
      console.log('parsedData:', parsedData);
    } catch (e) {
      console.log(`Caught error: ${e.message}`);
    }
  });
}).on('error', (e) => {
  console.log(`Got error: ${e.message}`);
});

... and every time he gets into the operator catchto: Caught error: Unexpected token in JSON at position 0. (Note the two spaces between the token and the c.)

I checked the JSON returned from both Chrome and Postman, with two different web JSON checks, and it returns as valid. When writing rawDatato a file, it looks like a buffer (?) ...

1fef bfbd 0800 0000 0000 0400 efbf bdef
bfbd efbf bd72 efbf bdc8 b62d efbf bd2b
0c3f 7547 1cef bfbd 00ef bfbd efbf bd0b
efbf bd5b 49ef bfbd 2def bfbd 6c6b efbf
bd5c 55ef bfbd efbf bd44 3fef bfbd 126c
71ef bfbd 021c 2029 6def bfbd 13ef bfbd
efbf bdef bfbd 437f 52ef bfbd 4227 48ef
bfbd efbf bd4d efbf bd31 13ef bfbd 09ef
bfbd 5d2f 7bef bfbd efbf bde5 aa81 745e
efbf bd65 efbf bd31 efbf bdef bfbd efbf
...

... Buffer.isBufferreturns false.

JSON.stringify ing, toString ing, new Buffer, , .trim replace , .

?

---

EDIT: , JSON, Chrome Postman, , -, . curl URL- , JSON. , , JSON, .

Answer 1

, api.511.org gzip api, api_key. json.

:

var request = require('request');

var apiUrl = 'http://api.511.org/transit/vehiclemonitoring?api_key=${API_KEYS.API_KEY_511}&format=json&agency=sf-muni';
//apiUrl = 'http://ip.jsontest.com/';

var response = request({
    method: 'GET',
    uri: apiUrl,
    gzip: true
}, function(error, response, body) {
    //* workaround for issue with this particular apiUrl
    var firstChar = body.substring(0, 1);
    var firstCharCode = body.charCodeAt(0);
    if (firstCharCode == 65279) {
        console.log('First character "' + firstChar + '" (character code: ' + firstCharCode + ') is invalid so removing it.');
        body = body.substring(1);
    }
    //*/

    var parsedJson = JSON.parse(body);
    console.log('parsedJson: ', parsedJson);
});