The use of the ESP8266 in the world of IoT

User avatar
By mars000
#59384 Hi
I want to read a json file from a website and parse the contents. Ideally I want to read all the file however I've settled to just being ok in reading first 2000 characters at it contains the info I need (and I haven't been able to work out how to read the entire file). I then simply search for "air_temp" value as per the code below. What I have found is that the value returned from client.available() varies from over 2400 to 1418 and it appears to be random (from what I can gather). I don't know why it would be doing this - hoping someone can help me out. And while at it is there an elegant way I can read the entire contents into a String or char[] that I can parse. Code below:

Code: Select all#include <ESP8266WiFi.h>
#include <WiFiClient.h>
// #include <ArduinoJson.h>
#include <SimpleTimer.h>

#define MAX_BYTES_TO_READ 2000

int n = 0;
 
// WiFi information
const char ssid[] = "xxxxx";
const char pass[] = "xxxxx";

 
// Remote site information
const char http_site[] = "www.bom.gov.au";
const int http_port = 80;
 
// Pin definitions
const int LED_PIN = 5;
 
// Global variables
WiFiClient client;

// char line[5000];
// String stringOne;
SimpleTimer timer;
int counter = 0;

void setup() {
 
  // Set up serial console to read web page
  Serial.begin(115200);
  Serial.println("Starting Thing GET Example");
 
  // Set up LED for debugging
 // pinMode(LED_PIN, OUTPUT);
 
  // Connect to WiFi
  connectWiFi();
 
  // Attempt to connect to website

  // get_observed_temp();
  timer.setInterval (10000L, get_observed_temp);
}


void get_observed_temp()
{
 char c;
 String stringOne = "";
   if (!getPage() ) {
    Serial.println("GET request failed");
  } else { Serial.println ("Get request passed");
  n = 0;
  // Lets read the json web file !
  Serial.print("Client available = ");
  Serial.println(client.available());

   while ((client.available()) and (n < MAX_BYTES_TO_READ)) {
  // while (n < MAX_BYTES_TO_READ) {
    c = client.read();
    Serial.print(c);
  //read char by char HTTP request - just read first 2000 characters only
    if (n < MAX_BYTES_TO_READ) {
    stringOne = stringOne + c;
    n++; }
  }  // end while loop - finished reading json file (at least the part we are interested in

  // ======== If the server has disconnected, stop the client and WiFi =======
  if (!client.connected() ) {    // should do this at end of everything
    Serial.println("Client disconnected !");
   
    // Close socket and wait for disconnect from WiFi
    client.stop();
    if ( WiFi.status() != WL_DISCONNECTED ) {
      WiFi.disconnect(); }
   
    // Do nothing
    Serial.println("Finished Thing GET test");
    Serial.println("*********************************");
    Serial.println ("here n = ");
    Serial.println (n);   
   
   
  //  return (-20);
  } // end server disconnected code
 
   if  ((client.available()) and (n == MAX_BYTES_TO_READ)) {
   // if  (n == MAX_BYTES_TO_READ) {
    Serial.println();
    Serial.println ("===============================================");
    Serial.println ("===============================================");
 
     if (stringOne.indexOf("Essendon Airport") > 0)
     {
     
      Serial.println ("Essendon Airport file !");
      int string_position_data = stringOne.indexOf("ID");     // get string position of term "ID"
      Serial.print ("Postion of ID in array = ");
      Serial.println (string_position_data);
   
      int string_position_data_1 = stringOne.indexOf("air_temp"); // get string position of term "Main ID"
      Serial.print ("Postion of air_temp in array = ");
      Serial.println (string_position_data_1);

      String air_temp;
      air_temp = stringOne.substring(string_position_data_1+11, string_position_data_1+18);
   
      Serial.print ("air_temp = ");
      Serial.println (air_temp);
 
      int air_temp_int = air_temp.toInt();

      Serial.print ("Air temp as an Integer = ");
      Serial.println(air_temp_int);
   
      Serial.println ("===============================================");
      Serial.println ("===============================================");
      Serial.println ("===============================================");

     }
     else { Serial.println ("Essendon Airport not found in file"); }
   }
    client.stop();
    Serial.println();
    Serial.print ("COUNTER = "); 
    Serial.println (counter++);
    Serial.print ("N = "); 
    Serial.println (n);
  }
}



void loop() {
   
   timer.run();
}

 
// Attempt to connect to WiFi
void connectWiFi() {
 
  byte led_status = 0;
 
   WiFi.begin(ssid, pass);                           // Connect to WiFi network
    Serial.println("Waiting to connect to Wifi:");
    while (WiFi.status() != WL_CONNECTED) {           // Wait for board to connect to WiFi network
      delay(500);                                   
      Serial.print("."); 
  }
}

 
// Perform an HTTP GET request to a remote page

bool getPage() {

    if (!client.connect(http_site, http_port)) {
    Serial.println("Connection failed :-(");
    return false;
  }

  Serial.println("Connected to host - sending request...");
 
  client.print(String("GET /fwo/IDV60901/IDV60901.95866.json HTTP/1.1\r\n") +
               "Host: " + http_site + "\r\n" +
               "Connection: close\r\n\r\n");
 
  Serial.println("Request sent - waiting for reply...");
  int timeout = millis() + 5000;
  while(client.available() == 0) {
    if (timeout - millis() < 0) {
        Serial.println(">>> Client Timeout !");
        client.stop();
        return false;
    } // end if
  } // end while
  return true;
}


WHAT SERIAL THE OUTPUT LOOKS LIKE WHEN NOT ENOUGH READ IN:

Code: Select allConnected to host - sending request...
Request sent - waiting for reply...
Get request passed
Client available = 1418
HTTP/1.1 200 OK
Server: Apache/2.2.3 (Red Hat)
Last-Modified: Mon, 12 Dec 2016 05:21:30 GMT
ETag: "17d8273-20d58-5436f46b12a80"
Content-Type: application/json
Date: Mon, 12 Dec 2016 05:28:12 GMT
Transfer-Encoding:  chunked
Connection: close
Connection: Transfer-Encoding

00006000
{
   "observations": {
      "notice": [
         {
            "copyright": "Copyright Commonwealth of Australia 2016, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
            "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
            "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
            "feedback_url": "http://www.bom.gov.au/other/feedback"
         }
      ],
      "header": [
         {
            "refresh_message": "Issued at  4:21 pm EDT Monday 12 December 2016",
            "ID": "IDV60901",
            "main_ID": "IDV60900",
            "name": "Essendon Airport",
            "state_time_zone": "VIC",
            "time_zone": "EDT",
            "product_name": "Capital City Observations",
            "state": "Victoria"
         }
      ],
      "data": [
         {
            "sort_order": 0,
            "wmo": 95866,
            "name": "Essendon Airport",
            "history_product": "IDV60901",
            "local_date_time": "12/04:00pm",
            "local_date_time_full": "20161212160000",
            "aifstime_utc": "20161212050000",
            "lat": -37.7,
            "lon": 144.9,
            "apparent_t": 26.0,
            "cloud": "-",
            "cloud_base_m": null,
            "cloud_oktas": nu
COUNTER = 728
N = 1418



WHAT SERIAL THE OUTPUT LOOKS LIKE WHEN ALL 2000 BYTES READ IN:
Code: Select allConnected to host - sending request...
Request sent - waiting for reply...
Get request passed
Client available = 5672
HTTP/1.1 200 OK
Server: Apache/2.2.3 (Red Hat)
Last-Modified: Mon, 12 Dec 2016 05:21:30 GMT
ETag: "17d8273-20d58-5436f46b12a80"
Content-Type: application/json
Date: Mon, 12 Dec 2016 05:27:22 GMT
Transfer-Encoding:  chunked
Connection: close
Connection: Transfer-Encoding

00006000
{
   "observations": {
      "notice": [
         {
            "copyright": "Copyright Commonwealth of Australia 2016, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
            "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
            "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
            "feedback_url": "http://www.bom.gov.au/other/feedback"
         }
      ],
      "header": [
         {
            "refresh_message": "Issued at  4:21 pm EDT Monday 12 December 2016",
            "ID": "IDV60901",
            "main_ID": "IDV60900",
            "name": "Essendon Airport",
            "state_time_zone": "VIC",
            "time_zone": "EDT",
            "product_name": "Capital City Observations",
            "state": "Victoria"
         }
      ],
      "data": [
         {
            "sort_order": 0,
            "wmo": 95866,
            "name": "Essendon Airport",
            "history_product": "IDV60901",
            "local_date_time": "12/04:00pm",
            "local_date_time_full": "20161212160000",
            "aifstime_utc": "20161212050000",
            "lat": -37.7,
            "lon": 144.9,
            "apparent_t": 26.0,
            "cloud": "-",
            "cloud_base_m": null,
            "cloud_oktas": null,
            "cloud_type_id": null,
            "cloud_type": "-",
            "delta_t": 15.4,
            "gust_kmh": 37,
            "gust_kt": 20,
            "air_temp": 32.3,
            "dewpt": 1.4,
            "press": 1014.4,
            "press_qnh": 1014.5,
            "press_msl": 1014.4,
            "press_tend": "-",
            "rain_trace": "0.0",
            "rel_hum": 14,
            "sea_state": "-",
            "swell_dir_worded": "-",
            "swell_height": null,
            "swell_period": null,
            "vis_km": "10",
            "weather": "-",
            "wind_dir": "NNW",
            "wind_spd_kmh": 24,
            "wind_spd_kt": 13
         },
         {
            "sort_order": 1,
            "wmo": 95866,
            "name": "Essendon Airpor
===============================================
===============================================
Essendon Airport file !
Postion of ID in array = 836
Postion of air_temp in array = 1537
air_temp = 32.3,
   
Air temp as an Integer = 32
===============================================
===============================================
===============================================

COUNTER = 723
N = 2000