Skip to Content

Street Scraping

Ever wanted to download Google Street View panoramas programmatically? If you said "yes!", then this is the code for you.

I reverse-engineered their URL query scheme using the Live HTTP Headers plug-in for Firefox. From there, I just parsed through the XML returned to download each jpeg tile and then stitch them together in order.

I used a variation of this code in combination with a few other programs to parse a GPS log, download the street view panorama for each position in the log, and then stitch each frame together into a video documenting my trip on the M5 bus.

  1. import processing.core.*;
  2. import processing.net.*;
  3. import processing.xml.*;
  4.  
  5. // get a hold of this for the xml library
  6. // better way to do this part?
  7. PApplet main = this;
  8.  
  9. void setup() {
  10.    // we usually end up with 3328 x 1664 pixel images
  11.    // let's undersample considerably for a more sane window size
  12.   size(832, 416);
  13.  
  14.   // your google maps api key goes here
  15.   // sign up: http://code.google.com/apis/maps/signup.html
  16.   String apiKey = "REPLACE ME WITH YOUR VERY OWN API KEY";
  17.  
  18.   // pick a location
  19.   // takes an address or a lat / lon
  20.   String location = "721 broadway, new york, ny";
  21.  
  22.   // generate the panorama
  23.   Panorama pano = new Panorama(location, apiKey);
  24.  
  25.   // show the image
  26.   image(pano.fullPano, 0, 0, width, height);
  27.  
  28.   // save the full-res image to the sketch folder
  29.   pano.fullPano.save(location + ".jpg");
  30.  
  31.   // skip the draw loop
  32.   noLoop();
  33. }
  34.  
  35.  
  36. class Panorama {
  37.   float lat;
  38.   float lon;
  39.   String panoId;  
  40.   int imageWidth;
  41.   int imageHeight;
  42.   int tileWidth;
  43.   int tileHeight;
  44.   int xTileCount;
  45.   int yTileCount;  
  46.   PImage[][] tiles;
  47.   PImage fullPano;
  48.   String apiKey;
  49.  
  50.   // constructor
  51.   Panorama(String address, String _apiKey) {
  52.     apiKey = _apiKey;
  53.     String[] location = geocode(address);
  54.     lon = float(location[0]);
  55.     lat = float(location[1]);
  56.  
  57.     fetchInfo(lat, lon);
  58.     buildImage();
  59.   }
  60.  
  61.  
  62.   void fetchInfo(float lat, float lon) {
  63.     String url = "http://maps.google.com/cbk?output=xml&ll=" + lat + "," + lon;
  64.  
  65.     XMLElement response = new XMLElement(main, url);
  66.  
  67.     XMLElement kid = response.getChild(0);
  68.     imageWidth = kid.getIntAttribute("image_width");
  69.     imageHeight = kid.getIntAttribute("image_height");
  70.     tileWidth = kid.getIntAttribute("tile_width");    
  71.     tileHeight = kid.getIntAttribute("tile_height");
  72.     panoId = kid.getStringAttribute("pano_id");
  73.  
  74.     xTileCount = imageWidth / tileWidth;
  75.     yTileCount = imageHeight / tileHeight;
  76.  
  77.     println("Panorama ID: " + panoId);
  78.     println("X Tile Count: " + xTileCount);
  79.     println("Y Tile Count: " + yTileCount);
  80.   }
  81.  
  82.   void buildImage() {
  83.     fullPano = createImage(imageWidth, imageHeight, RGB);
  84.     fullPano.loadPixels();
  85.  
  86.     for(int xPos = 0; xPos <= xTileCount; xPos++) {
  87.       for(int yPos = 0; yPos <= yTileCount; yPos++) {
  88.         // &.jpg fools processing into handling it
  89.         String imageUrl = "http://cbk0.google.com/cbk?output=tile&panoid=" + panoId + "&zoom=3&x=" + xPos + "&y=" + yPos + "&.jpg";
  90.         fullPano.set(xPos * tileWidth, yPos * tileHeight, loadImage(imageUrl));
  91.         println("Loaded tile " + xPos + "::" + yPos);      
  92.       }
  93.     }
  94.  
  95.     fullPano.updatePixels();        
  96.   }
  97.  
  98.   String[] geocode(String address) {
  99.     String cleanAddress =  address.replace(' ', '+');
  100.     String url = "http://maps.google.com/maps/geo?q=" + cleanAddress + "&output=xml&oe=utf8&sensor=false&key=" + apiKey;
  101.     println(url);
  102.     XMLElement location = new XMLElement(main, url);
  103.     XMLElement kid = location.getChild("Response/Placemark/Point/coordinates");
  104.     String rawCoordinates = kid.getContent();
  105.     String[] latlon = shorten(split(rawCoordinates, ',')); // ditch the 0
  106.  
  107.     println(latlon);
  108.  
  109.     return latlon;
  110.   }
  111. }

October 7 2009 at 4 AM

Add Your Comment

Previous in chronology
Lab: Servo Motor Control
Previous in course
M5
Next
Street Level