001 // License: GPL. For details, see LICENSE file.
002 package org.openstreetmap.josm.data.imagery;
003
004
005 import java.awt.Graphics2D;
006 import java.awt.image.BufferedImage;
007 import java.io.BufferedOutputStream;
008 import java.io.File;
009 import java.io.FileInputStream;
010 import java.io.FileNotFoundException;
011 import java.io.FileOutputStream;
012 import java.io.IOException;
013 import java.io.InputStream;
014 import java.io.OutputStream;
015 import java.lang.ref.SoftReference;
016 import java.net.URLConnection;
017 import java.text.SimpleDateFormat;
018 import java.util.ArrayList;
019 import java.util.Calendar;
020 import java.util.Collections;
021 import java.util.Comparator;
022 import java.util.HashMap;
023 import java.util.HashSet;
024 import java.util.Iterator;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.Properties;
028 import java.util.Set;
029
030 import javax.imageio.ImageIO;
031 import javax.xml.bind.JAXBContext;
032 import javax.xml.bind.Marshaller;
033 import javax.xml.bind.Unmarshaller;
034
035 import org.openstreetmap.josm.Main;
036 import org.openstreetmap.josm.data.ProjectionBounds;
037 import org.openstreetmap.josm.data.coor.EastNorth;
038 import org.openstreetmap.josm.data.coor.LatLon;
039 import org.openstreetmap.josm.data.imagery.types.EntryType;
040 import org.openstreetmap.josm.data.imagery.types.ProjectionType;
041 import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
042 import org.openstreetmap.josm.data.preferences.StringProperty;
043 import org.openstreetmap.josm.data.projection.Projection;
044 import org.openstreetmap.josm.gui.NavigatableComponent;
045 import org.openstreetmap.josm.tools.Utils;
046
047
048
049 public class WmsCache {
050 //TODO Property for maximum cache size
051 //TODO Property for maximum age of tile, automatically remove old tiles
052 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
053 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
054
055 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
056 private static final String INDEX_FILENAME = "index.xml";
057 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
058
059 private static class CacheEntry {
060 final double pixelPerDegree;
061 final double east;
062 final double north;
063 final ProjectionBounds bounds;
064 final String filename;
065
066 long lastUsed;
067 long lastModified;
068
069 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
070 this.pixelPerDegree = pixelPerDegree;
071 this.east = east;
072 this.north = north;
073 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
074 this.filename = filename;
075 }
076 }
077
078 private static class ProjectionEntries {
079 final String projection;
080 final String cacheDirectory;
081 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
082
083 ProjectionEntries(String projection, String cacheDirectory) {
084 this.projection = projection;
085 this.cacheDirectory = cacheDirectory;
086 }
087 }
088
089 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
090 private final File cacheDir;
091 private final int tileSize; // Should be always 500
092 private int totalFileSize;
093 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
094 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
095 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
096 private Set<ProjectionBounds> areaToCache;
097
098 protected String cacheDirPath() {
099 String cPath = PROP_CACHE_PATH.get();
100 if (!(new File(cPath).isAbsolute())) {
101 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
102 }
103 // Migrate to new cache directory. Remove 2012-06
104 {
105 File oldPath = new File(Main.pref.getPreferencesDirFile(), "wms-cache");
106 File newPath = new File(cPath);
107 if (oldPath.exists() && !newPath.exists()) {
108 oldPath.renameTo(newPath);
109 }
110 }
111 return cPath;
112 }
113
114 public WmsCache(String url, int tileSize) {
115 File globalCacheDir = new File(cacheDirPath());
116 globalCacheDir.mkdirs();
117 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
118 cacheDir.mkdirs();
119 this.tileSize = tileSize;
120 }
121
122 private String getCacheDirectory(String url) {
123 String cacheDirName = null;
124 InputStream fis = null;
125 OutputStream fos = null;
126 try {
127 Properties layersIndex = new Properties();
128 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
129 try {
130 fis = new FileInputStream(layerIndexFile);
131 layersIndex.load(fis);
132 } catch (FileNotFoundException e) {
133 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
134 } catch (IOException e) {
135 System.err.println("Unable to load layers index for wms cache");
136 e.printStackTrace();
137 }
138
139 for (Object propKey: layersIndex.keySet()) {
140 String s = (String)propKey;
141 if (url.equals(layersIndex.getProperty(s))) {
142 cacheDirName = s;
143 break;
144 }
145 }
146
147 if (cacheDirName == null) {
148 int counter = 0;
149 while (true) {
150 counter++;
151 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
152 break;
153 }
154 }
155 cacheDirName = String.valueOf(counter);
156 layersIndex.setProperty(cacheDirName, url);
157 try {
158 fos = new FileOutputStream(layerIndexFile);
159 layersIndex.store(fos, "");
160 } catch (IOException e) {
161 System.err.println("Unable to save layer index for wms cache");
162 e.printStackTrace();
163 }
164 }
165 } finally {
166 try {
167 if (fis != null) {
168 fis.close();
169 }
170 if (fos != null) {
171 fos.close();
172 }
173 } catch (IOException e) {
174 e.printStackTrace();
175 }
176 }
177
178 return cacheDirName;
179 }
180
181 private ProjectionEntries getProjectionEntries(Projection projection) {
182 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
183 }
184
185 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
186 ProjectionEntries result = entries.get(projection);
187 if (result == null) {
188 result = new ProjectionEntries(projection, cacheDirectory);
189 entries.put(projection, result);
190 }
191
192 return result;
193 }
194
195 public synchronized void loadIndex() {
196 File indexFile = new File(cacheDir, INDEX_FILENAME);
197 try {
198 JAXBContext context = JAXBContext.newInstance(
199 WmsCacheType.class.getPackage().getName(),
200 WmsCacheType.class.getClassLoader());
201 Unmarshaller unmarshaller = context.createUnmarshaller();
202 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
203 totalFileSize = cacheEntries.getTotalFileSize();
204 if (cacheEntries.getTileSize() != tileSize) {
205 System.out.println("Cache created with different tileSize, cache will be discarded");
206 return;
207 }
208 for (ProjectionType projectionType: cacheEntries.getProjection()) {
209 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
210 for (EntryType entry: projectionType.getEntry()) {
211 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
212 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
213 ce.lastModified = entry.getLastModified().getTimeInMillis();
214 projection.entries.add(ce);
215 }
216 }
217 } catch (Exception e) {
218 if (indexFile.exists()) {
219 e.printStackTrace();
220 System.out.println("Unable to load index for wms-cache, new file will be created");
221 } else {
222 System.out.println("Index for wms-cache doesn't exist, new file will be created");
223 }
224 }
225
226 removeNonReferencedFiles();
227 }
228
229 private void removeNonReferencedFiles() {
230
231 Set<String> usedProjections = new HashSet<String>();
232
233 for (ProjectionEntries projectionEntries: entries.values()) {
234
235 usedProjections.add(projectionEntries.cacheDirectory);
236
237 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
238 if (projectionDir.exists()) {
239 Set<String> referencedFiles = new HashSet<String>();
240
241 for (CacheEntry ce: projectionEntries.entries) {
242 referencedFiles.add(ce.filename);
243 }
244
245 for (File file: projectionDir.listFiles()) {
246 if (!referencedFiles.contains(file.getName())) {
247 file.delete();
248 }
249 }
250 }
251 }
252
253 for (File projectionDir: cacheDir.listFiles()) {
254 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
255 Utils.deleteDirectory(projectionDir);
256 }
257 }
258 }
259
260 private int calculateTotalFileSize() {
261 int result = 0;
262 for (ProjectionEntries projectionEntries: entries.values()) {
263 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
264 while (it.hasNext()) {
265 CacheEntry entry = it.next();
266 File imageFile = getImageFile(projectionEntries, entry);
267 if (!imageFile.exists()) {
268 it.remove();
269 } else {
270 result += imageFile.length();
271 }
272 }
273 }
274 return result;
275 }
276
277 public synchronized void saveIndex() {
278 WmsCacheType index = new WmsCacheType();
279
280 if (totalFileSizeDirty) {
281 totalFileSize = calculateTotalFileSize();
282 }
283
284 index.setTileSize(tileSize);
285 index.setTotalFileSize(totalFileSize);
286 for (ProjectionEntries projectionEntries: entries.values()) {
287 if (projectionEntries.entries.size() > 0) {
288 ProjectionType projectionType = new ProjectionType();
289 projectionType.setName(projectionEntries.projection);
290 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
291 index.getProjection().add(projectionType);
292 for (CacheEntry ce: projectionEntries.entries) {
293 EntryType entry = new EntryType();
294 entry.setPixelPerDegree(ce.pixelPerDegree);
295 entry.setEast(ce.east);
296 entry.setNorth(ce.north);
297 Calendar c = Calendar.getInstance();
298 c.setTimeInMillis(ce.lastUsed);
299 entry.setLastUsed(c);
300 c = Calendar.getInstance();
301 c.setTimeInMillis(ce.lastModified);
302 entry.setLastModified(c);
303 entry.setFilename(ce.filename);
304 projectionType.getEntry().add(entry);
305 }
306 }
307 }
308 try {
309 JAXBContext context = JAXBContext.newInstance(
310 WmsCacheType.class.getPackage().getName(),
311 WmsCacheType.class.getClassLoader());
312 Marshaller marshaller = context.createMarshaller();
313 marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME)));
314 } catch (Exception e) {
315 System.err.println("Failed to save wms-cache file");
316 e.printStackTrace();
317 }
318 }
319
320 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
321 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
322 }
323
324
325 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
326
327 synchronized (this) {
328 entry.lastUsed = System.currentTimeMillis();
329
330 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
331 if (memCache != null) {
332 BufferedImage result = memCache.get();
333 if (result != null)
334 return result;
335 }
336 }
337
338 try {
339 // Reading can't be in synchronized section, it's too slow
340 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
341 synchronized (this) {
342 if (result == null) {
343 projectionEntries.entries.remove(entry);
344 totalFileSizeDirty = true;
345 }
346 return result;
347 }
348 } catch (IOException e) {
349 synchronized (this) {
350 projectionEntries.entries.remove(entry);
351 totalFileSizeDirty = true;
352 throw e;
353 }
354 }
355 }
356
357 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
358 for (CacheEntry entry: projectionEntries.entries) {
359 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
360 return entry;
361 }
362 return null;
363 }
364
365 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
366 ProjectionEntries projectionEntries = getProjectionEntries(projection);
367 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
368 return (entry != null);
369 }
370
371 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
372 CacheEntry entry = null;
373 ProjectionEntries projectionEntries = null;
374 synchronized (this) {
375 projectionEntries = getProjectionEntries(projection);
376 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
377 }
378 if (entry != null) {
379 try {
380 return loadImage(projectionEntries, entry);
381 } catch (IOException e) {
382 System.err.println("Unable to load file from wms cache");
383 e.printStackTrace();
384 return null;
385 }
386 }
387 return null;
388 }
389
390 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
391 ProjectionEntries projectionEntries;
392 List<CacheEntry> matches;
393 synchronized (this) {
394 matches = new ArrayList<WmsCache.CacheEntry>();
395
396 double minPPD = pixelPerDegree / 5;
397 double maxPPD = pixelPerDegree * 5;
398 projectionEntries = getProjectionEntries(projection);
399
400 double size2 = tileSize / pixelPerDegree;
401 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
402 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
403 east + size2 - border, north + size2 - border);
404
405 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
406 for (CacheEntry entry: projectionEntries.entries) {
407 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
408 entry.lastUsed = System.currentTimeMillis();
409 matches.add(entry);
410 }
411 }
412
413 if (matches.isEmpty())
414 return null;
415
416
417 Collections.sort(matches, new Comparator<CacheEntry>() {
418 @Override
419 public int compare(CacheEntry o1, CacheEntry o2) {
420 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
421 }
422 });
423 }
424
425 //TODO Use alpha layer only when enabled on wms layer
426 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
427 Graphics2D g = result.createGraphics();
428
429
430 boolean drawAtLeastOnce = false;
431 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
432 for (CacheEntry ce: matches) {
433 BufferedImage img;
434 try {
435 img = loadImage(projectionEntries, ce);
436 localCache.put(ce, new SoftReference<BufferedImage>(img));
437 } catch (IOException e) {
438 continue;
439 }
440
441 drawAtLeastOnce = true;
442
443 int xDiff = (int)((ce.east - east) * pixelPerDegree);
444 int yDiff = (int)((ce.north - north) * pixelPerDegree);
445 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
446
447 int x = xDiff;
448 int y = -size + tileSize - yDiff;
449
450 g.drawImage(img, x, y, size, size, null);
451 }
452
453 if (drawAtLeastOnce) {
454 synchronized (this) {
455 memoryCache.putAll(localCache);
456 }
457 return result;
458 } else
459 return null;
460 }
461
462 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
463 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
464 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
465 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
466
467 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
468 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
469 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
470 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
471
472 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
473 String extension;
474 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
475 extension = "jpg";
476 } else if ("image/png".equals(mimeType)) {
477 extension = "png";
478 } else if ("image/gif".equals(mimeType)) {
479 extension = "gif";
480 } else {
481 extension = "dat";
482 }
483
484 int counter = 0;
485 FILENAME_LOOP:
486 while (true) {
487 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
488 for (CacheEntry entry: projectionEntries.entries) {
489 if (entry.filename.equals(result)) {
490 counter++;
491 continue FILENAME_LOOP;
492 }
493 }
494 return result;
495 }
496 }
497
498 /**
499 *
500 * @param img Used only when overlapping is used, when not used, used raw from imageData
501 * @param imageData
502 * @param projection
503 * @param pixelPerDegree
504 * @param east
505 * @param north
506 * @throws IOException
507 */
508 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
509 ProjectionEntries projectionEntries = getProjectionEntries(projection);
510 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
511 File imageFile;
512 if (entry == null) {
513
514 String mimeType;
515 if (img != null) {
516 mimeType = "image/png";
517 } else {
518 mimeType = URLConnection.guessContentTypeFromStream(imageData);
519 }
520 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
521 entry.lastUsed = System.currentTimeMillis();
522 entry.lastModified = entry.lastUsed;
523 projectionEntries.entries.add(entry);
524 imageFile = getImageFile(projectionEntries, entry);
525 } else {
526 imageFile = getImageFile(projectionEntries, entry);
527 totalFileSize -= imageFile.length();
528 }
529
530 imageFile.getParentFile().mkdirs();
531
532 if (img != null) {
533 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
534 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
535 ImageIO.write(copy, "png", imageFile);
536 totalFileSize += imageFile.length();
537 } else {
538 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
539 try {
540 totalFileSize += Utils.copyStream(imageData, os);
541 } finally {
542 os.close();
543 }
544 }
545 }
546
547 public synchronized void cleanSmallFiles(int size) {
548 for (ProjectionEntries projectionEntries: entries.values()) {
549 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
550 while (it.hasNext()) {
551 File file = getImageFile(projectionEntries, it.next());
552 long length = file.length();
553 if (length <= size) {
554 if (length == 0) {
555 totalFileSizeDirty = true; // File probably doesn't exist
556 }
557 totalFileSize -= size;
558 file.delete();
559 it.remove();
560 }
561 }
562 }
563 }
564
565 public static String printDate(Calendar c) {
566 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
567 }
568
569 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
570 for (ProjectionBounds b: areaToCache) {
571 if (cacheEntry.bounds.intersects(b))
572 return true;
573 }
574 return false;
575 }
576
577 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
578 this.areaToCache = areaToCache;
579 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
580 while (it.hasNext()) {
581 if (!isInsideAreaToCache(it.next())) {
582 it.remove();
583 }
584 }
585 }
586 }