001 package org.openstreetmap.gui.jmapviewer;
002
003 //License: GPL. Copyright 2008 by Jan Peter Stotz
004
005 import java.io.BufferedReader;
006 import java.io.ByteArrayInputStream;
007 import java.io.ByteArrayOutputStream;
008 import java.io.File;
009 import java.io.FileInputStream;
010 import java.io.FileNotFoundException;
011 import java.io.FileOutputStream;
012 import java.io.IOException;
013 import java.io.InputStream;
014 import java.io.InputStreamReader;
015 import java.io.OutputStreamWriter;
016 import java.io.PrintWriter;
017 import java.lang.Thread;
018 import java.net.HttpURLConnection;
019 import java.net.URL;
020 import java.net.URLConnection;
021 import java.nio.charset.Charset;
022 import java.util.HashMap;
023 import java.util.Map;
024 import java.util.Map.Entry;
025 import java.util.Random;
026 import java.util.logging.Level;
027 import java.util.logging.Logger;
028
029 import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
030 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
031 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
032 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
033 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;
034
035 /**
036 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and
037 * saves all loaded files in a directory located in the temporary directory.
038 * If a tile is present in this file cache it will not be loaded from OSM again.
039 *
040 * @author Jan Peter Stotz
041 * @author Stefan Zeller
042 */
043 public class OsmFileCacheTileLoader extends OsmTileLoader {
044
045 private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName());
046
047 private static final String ETAG_FILE_EXT = ".etag";
048 private static final String TAGS_FILE_EXT = ".tags";
049
050 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");
051
052 public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24;
053 public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7;
054
055 protected String cacheDirBase;
056
057 protected final Map<TileSource, File> sourceCacheDirMap;
058
059 protected long maxCacheFileAge = FILE_AGE_ONE_WEEK;
060 protected long recheckAfter = FILE_AGE_ONE_DAY;
061
062 public static File getDefaultCacheDir() throws SecurityException {
063 String tempDir = null;
064 String userName = System.getProperty("user.name");
065 try {
066 tempDir = System.getProperty("java.io.tmpdir");
067 } catch (SecurityException e) {
068 log.log(Level.WARNING,
069 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "
070 + e.toString());
071 throw e; // rethrow
072 }
073 try {
074 if (tempDir == null)
075 throw new IOException("No temp directory set");
076 String subDirName = "JMapViewerTiles";
077 // On Linux/Unix systems we do not have a per user tmp directory.
078 // Therefore we add the user name for getting a unique dir name.
079 if (userName != null && userName.length() > 0) {
080 subDirName += "_" + userName;
081 }
082 File cacheDir = new File(tempDir, subDirName);
083 return cacheDir;
084 } catch (Exception e) {
085 }
086 return null;
087 }
088
089 /**
090 * Create a OSMFileCacheTileLoader with given cache directory.
091 * If cacheDir is not set or invalid, IOException will be thrown.
092 * @param map the listener checking for tile load events (usually the map for display)
093 * @param cacheDir directory to store cached tiles
094 */
095 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException {
096 super(map);
097 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))
098 throw new IOException("Cannot access cache directory");
099
100 log.finest("Tile cache directory: " + cacheDir);
101 cacheDirBase = cacheDir.getAbsolutePath();
102 sourceCacheDirMap = new HashMap<TileSource, File>();
103 }
104
105 /**
106 * Create a OSMFileCacheTileLoader with system property temp dir.
107 * If not set an IOException will be thrown.
108 * @param map the listener checking for tile load events (usually the map for display)
109 */
110 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {
111 this(map, getDefaultCacheDir());
112 }
113
114 @Override
115 public TileJob createTileLoaderJob(final Tile tile) {
116 return new FileLoadJob(tile);
117 }
118
119 protected File getSourceCacheDir(TileSource source) {
120 File dir = sourceCacheDirMap.get(source);
121 if (dir == null) {
122 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));
123 if (!dir.exists()) {
124 dir.mkdirs();
125 }
126 }
127 return dir;
128 }
129
130 protected class FileLoadJob implements TileJob {
131 InputStream input = null;
132
133 Tile tile;
134 File tileCacheDir;
135 File tileFile = null;
136 long fileAge = 0;
137 boolean fileTilePainted = false;
138
139 public FileLoadJob(Tile tile) {
140 this.tile = tile;
141 }
142
143 public Tile getTile() {
144 return tile;
145 }
146
147 public void run() {
148 synchronized (tile) {
149 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())
150 return;
151 tile.loaded = false;
152 tile.error = false;
153 tile.loading = true;
154 }
155 tileCacheDir = getSourceCacheDir(tile.getSource());
156 if (loadTileFromFile()) {
157 return;
158 }
159 if (fileTilePainted) {
160 TileJob job = new TileJob() {
161
162 public void run() {
163 loadOrUpdateTile();
164 }
165 public Tile getTile() {
166 return tile;
167 }
168 };
169 JobDispatcher.getInstance().addJob(job);
170 } else {
171 loadOrUpdateTile();
172 }
173 }
174
175 protected void loadOrUpdateTile() {
176 try {
177 URLConnection urlConn = loadTileFromOsm(tile);
178 if (tileFile != null) {
179 switch (tile.getSource().getTileUpdate()) {
180 case IfModifiedSince:
181 urlConn.setIfModifiedSince(fileAge);
182 break;
183 case LastModified:
184 if (!isOsmTileNewer(fileAge)) {
185 log.finest("LastModified test: local version is up to date: " + tile);
186 tile.setLoaded(true);
187 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
188 return;
189 }
190 break;
191 }
192 }
193 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {
194 String fileETag = tile.getValue("etag");
195 if (fileETag != null) {
196 switch (tile.getSource().getTileUpdate()) {
197 case IfNoneMatch:
198 urlConn.addRequestProperty("If-None-Match", fileETag);
199 break;
200 case ETag:
201 if (hasOsmTileETag(fileETag)) {
202 tile.setLoaded(true);
203 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge
204 + recheckAfter);
205 return;
206 }
207 }
208 }
209 tile.putValue("etag", urlConn.getHeaderField("ETag"));
210 }
211 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
212 // If we are isModifiedSince or If-None-Match has been set
213 // and the server answers with a HTTP 304 = "Not Modified"
214 log.finest("ETag test: local version is up to date: " + tile);
215 tile.setLoaded(true);
216 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
217 return;
218 }
219
220 loadTileMetadata(tile, urlConn);
221 saveTagsToFile();
222
223 if ("no-tile".equals(tile.getValue("tile-info")))
224 {
225 tile.setError("No tile at this zoom level");
226 listener.tileLoadingFinished(tile, true);
227 } else {
228 for(int i = 0; i < 5; ++i) {
229 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
230 Thread.sleep(5000+(new Random()).nextInt(5000));
231 continue;
232 }
233 byte[] buffer = loadTileInBuffer(urlConn);
234 if (buffer != null) {
235 tile.loadImage(new ByteArrayInputStream(buffer));
236 tile.setLoaded(true);
237 listener.tileLoadingFinished(tile, true);
238 saveTileToFile(buffer);
239 break;
240 }
241 }
242 }
243 } catch (Exception e) {
244 tile.setError(e.getMessage());
245 listener.tileLoadingFinished(tile, false);
246 if (input == null) {
247 try {
248 System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage());
249 } catch(IOException i) {
250 }
251 }
252 } finally {
253 tile.loading = false;
254 tile.setLoaded(true);
255 }
256 }
257
258 protected boolean loadTileFromFile() {
259 FileInputStream fin = null;
260 try {
261 tileFile = getTileFile();
262 if (!tileFile.exists())
263 return false;
264
265 loadTagsFromFile();
266 if ("no-tile".equals(tile.getValue("tile-info")))
267 {
268 tile.setError("No tile at this zoom level");
269 if (tileFile.exists()) {
270 tileFile.delete();
271 }
272 tileFile = getTagsFile();
273 } else {
274 fin = new FileInputStream(tileFile);
275 if (fin.available() == 0)
276 throw new IOException("File empty");
277 tile.loadImage(fin);
278 fin.close();
279 }
280
281 fileAge = tileFile.lastModified();
282 boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge;
283 if (!oldTile) {
284 tile.setLoaded(true);
285 listener.tileLoadingFinished(tile, true);
286 fileTilePainted = true;
287 return true;
288 }
289 listener.tileLoadingFinished(tile, true);
290 fileTilePainted = true;
291 } catch (Exception e) {
292 try {
293 if (fin != null) {
294 fin.close();
295 tileFile.delete();
296 }
297 } catch (Exception e1) {
298 }
299 tileFile = null;
300 fileAge = 0;
301 }
302 return false;
303 }
304
305 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {
306 input = urlConn.getInputStream();
307 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
308 byte[] buffer = new byte[2048];
309 boolean finished = false;
310 do {
311 int read = input.read(buffer);
312 if (read >= 0) {
313 bout.write(buffer, 0, read);
314 } else {
315 finished = true;
316 }
317 } while (!finished);
318 if (bout.size() == 0)
319 return null;
320 return bout.toByteArray();
321 }
322
323 /**
324 * Performs a <code>HEAD</code> request for retrieving the
325 * <code>LastModified</code> header value.
326 *
327 * Note: This does only work with servers providing the
328 * <code>LastModified</code> header:
329 * <ul>
330 * <li>{@link tilesources.OsmTileSource.CycleMap} - supported</li>
331 * <li>{@link tilesources.OsmTileSource.Mapnik} - not supported</li>
332 * </ul>
333 *
334 * @param fileAge time of the
335 * @return <code>true</code> if the tile on the server is newer than the
336 * file
337 * @throws IOException
338 */
339 protected boolean isOsmTileNewer(long fileAge) throws IOException {
340 URL url;
341 url = new URL(tile.getUrl());
342 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
343 prepareHttpUrlConnection(urlConn);
344 urlConn.setRequestMethod("HEAD");
345 urlConn.setReadTimeout(30000); // 30 seconds read timeout
346 // System.out.println("Tile age: " + new
347 // Date(urlConn.getLastModified()) + " / "
348 // + new Date(fileAge));
349 long lastModified = urlConn.getLastModified();
350 if (lastModified == 0)
351 return true; // no LastModified time returned
352 return (lastModified > fileAge);
353 }
354
355 protected boolean hasOsmTileETag(String eTag) throws IOException {
356 URL url;
357 url = new URL(tile.getUrl());
358 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
359 prepareHttpUrlConnection(urlConn);
360 urlConn.setRequestMethod("HEAD");
361 urlConn.setReadTimeout(30000); // 30 seconds read timeout
362 // System.out.println("Tile age: " + new
363 // Date(urlConn.getLastModified()) + " / "
364 // + new Date(fileAge));
365 String osmETag = urlConn.getHeaderField("ETag");
366 if (osmETag == null)
367 return true;
368 return (osmETag.equals(eTag));
369 }
370
371 protected File getTileFile() {
372 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
373 + tile.getSource().getTileType());
374 }
375
376 protected File getTagsFile() {
377 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile()
378 + TAGS_FILE_EXT);
379 }
380
381 protected void saveTileToFile(byte[] rawData) {
382 try {
383 FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile()
384 + "_" + tile.getYtile() + "." + tile.getSource().getTileType());
385 f.write(rawData);
386 f.close();
387 // System.out.println("Saved tile to file: " + tile);
388 } catch (Exception e) {
389 System.err.println("Failed to save tile content: " + e.getLocalizedMessage());
390 }
391 }
392
393 protected void saveTagsToFile() {
394 File tagsFile = getTagsFile();
395 if (tile.getMetadata() == null) {
396 tagsFile.delete();
397 return;
398 }
399 try {
400 final PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile),
401 TAGS_CHARSET));
402 for (Entry<String, String> entry : tile.getMetadata().entrySet()) {
403 f.println(entry.getKey() + "=" + entry.getValue());
404 }
405 f.close();
406 } catch (Exception e) {
407 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());
408 }
409 }
410
411 /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/
412 private void loadOldETagfromFile() {
413 File etagFile = new File(tileCacheDir, tile.getZoom() + "_"
414 + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT);
415 if (!etagFile.exists()) return;
416 try {
417 FileInputStream f = new FileInputStream(etagFile);
418 byte[] buf = new byte[f.available()];
419 f.read(buf);
420 f.close();
421 String etag = new String(buf, TAGS_CHARSET.name());
422 tile.putValue("etag", etag);
423 if (etagFile.delete()) {
424 saveTagsToFile();
425 }
426 } catch (IOException e) {
427 System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage());
428 }
429 }
430
431 protected void loadTagsFromFile() {
432 loadOldETagfromFile();
433 File tagsFile = getTagsFile();
434 try {
435 final BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile),
436 TAGS_CHARSET));
437 for (String line = f.readLine(); line != null; line = f.readLine()) {
438 final int i = line.indexOf('=');
439 if (i == -1 || i == 0) {
440 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);
441 continue;
442 }
443 tile.putValue(line.substring(0,i),line.substring(i+1));
444 }
445 f.close();
446 } catch (FileNotFoundException e) {
447 } catch (Exception e) {
448 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());
449 }
450 }
451
452 }
453
454 public long getMaxFileAge() {
455 return maxCacheFileAge;
456 }
457
458 /**
459 * Sets the maximum age of the local cached tile in the file system. If a
460 * local tile is older than the specified file age
461 * {@link OsmFileCacheTileLoader} will connect to the tile server and check
462 * if a newer tile is available using the mechanism specified for the
463 * selected tile source/server.
464 *
465 * @param maxFileAge
466 * maximum age in milliseconds
467 * @see #FILE_AGE_ONE_DAY
468 * @see #FILE_AGE_ONE_WEEK
469 * @see TileSource#getTileUpdate()
470 */
471 public void setCacheMaxFileAge(long maxFileAge) {
472 this.maxCacheFileAge = maxFileAge;
473 }
474
475 public String getCacheDirBase() {
476 return cacheDirBase;
477 }
478
479 public void setTileCacheDir(String tileCacheDir) {
480 File dir = new File(tileCacheDir);
481 dir.mkdirs();
482 this.cacheDirBase = dir.getAbsolutePath();
483 }
484
485 public static interface TileClearController {
486
487 void initClearDir(File dir);
488
489 void initClearFiles(File[] files);
490
491 boolean cancel();
492
493 void fileDeleted(File file);
494
495 void clearFinished();
496 }
497
498 public void clearCache(TileSource source) {
499 clearCache(source, null);
500 }
501
502 public void clearCache(TileSource source, TileClearController controller) {
503 File dir = getSourceCacheDir(source);
504 if (dir != null) {
505 if (controller != null) controller.initClearDir(dir);
506 if (dir.isDirectory()) {
507 File[] files = dir.listFiles();
508 if (controller != null) controller.initClearFiles(files);
509 for (File file : files) {
510 if (controller != null && controller.cancel()) return;
511 file.delete();
512 if (controller != null) controller.fileDeleted(file);
513 }
514 }
515 dir.delete();
516 }
517 if (controller != null) controller.clearFinished();
518 }
519 }