Merge branch 'master' into libdeflate

systemed · web-flow · commit 5fa5fa31a09f · 2024-10-13T15:44:09.000+01:00
diff --git a/include/helpers.h b/include/helpers.h
@@ -7,6 +7,12 @@
 
 #define Z_DEFAULT_COMPRESSION -1
 
+#ifdef _MSVC_LANG
+#define ISATTY true
+#else
+#define ISATTY isatty(1)
+#endif
+
 // General helper routines
 
 inline void endian_swap(unsigned int& x) {
diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h
@@ -69,6 +69,8 @@ class OsmLuaProcessing {
 	// Do we have Lua routines for non-MP relations?
 	bool canReadRelations();
 	bool canPostScanRelations();
+	bool canWriteNodes();
+	bool canWriteWays();
 	bool canWriteRelations();
 
 	// Shapefile tag remapping
@@ -264,6 +266,8 @@ class OsmLuaProcessing {
 	bool supportsRemappingShapefiles;
 	bool supportsReadingRelations;
 	bool supportsPostScanRelations;
+	bool supportsWritingNodes;
+	bool supportsWritingWays;
 	bool supportsWritingRelations;
 	const class ShpMemTiles &shpMemTiles;
 	class OsmMemTiles &osmMemTiles;
diff --git a/include/tile_data.h b/include/tile_data.h
@@ -69,7 +69,7 @@ template<typename OO> void finalizeObjects(
 	int i = -1;
 	for (auto it = begin; it != end; it++) {
 		i++;
-		if (it->size() > 0 || i % 10 == 0 || i == 4095) {
+		if (it->size() > 0 || i % 50 == 0 || i == 4095) {
 			std::cout << "\r" << name << ": finalizing z6 tile " << (i + 1) << "/" << CLUSTER_ZOOM_AREA;
 
 #ifdef CLOCK_MONOTONIC
diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp
@@ -235,6 +235,8 @@ OsmLuaProcessing::OsmLuaProcessing(
 	supportsRemappingShapefiles = !!luaState["attribute_function"];
 	supportsReadingRelations    = !!luaState["relation_scan_function"];
 	supportsPostScanRelations   = !!luaState["relation_postscan_function"];
+	supportsWritingNodes        = !!luaState["node_function"];
+	supportsWritingWays         = !!luaState["way_function"];
 	supportsWritingRelations    = !!luaState["relation_function"];
 
 	// ---- Call init_function of Lua logic
@@ -272,6 +274,14 @@ bool OsmLuaProcessing::canPostScanRelations() {
 	return supportsPostScanRelations;
 }
 
+bool OsmLuaProcessing::canWriteNodes() {
+	return supportsWritingNodes;
+}
+
+bool OsmLuaProcessing::canWriteWays() {
+	return supportsWritingWays;
+}
+
 bool OsmLuaProcessing::canWriteRelations() {
 	return supportsWritingRelations;
 }
@@ -1064,7 +1074,10 @@ void OsmLuaProcessing::setRelation(
 	// Start Lua processing for relation
 	if (!isNativeMP && !supportsWritingRelations) return;
 	try {
-		luaState[isNativeMP ? "way_function" : "relation_function"]();
+		if (isNativeMP && supportsWritingWays)
+			luaState["way_function"]();
+		else if (!isNativeMP && supportsWritingRelations)
+			luaState["relation_function"]();
 	} catch(luaProcessingException &e) {
 		std::cerr << "Lua error on relation " << originalOsmID << std::endl;
 		exit(1);
diff --git a/src/pbf_processor.cpp b/src/pbf_processor.cpp
@@ -17,6 +17,11 @@ const std::string OptionSortTypeThenID = "Sort.Type_then_ID";
 const std::string OptionLocationsOnWays = "LocationsOnWays";
 std::atomic<uint64_t> blocksProcessed(0), blocksToProcess(0);
 
+// Access is guarded by ioMutex.
+// This counter decreases the chattiness of tilemaker's progress updates,
+// especially when run in a non-interactive context.
+uint64_t phaseProgress = 0;
+
 // Thread-local so that we can re-use buffers during parsing.
 thread_local PbfReader::PbfReader reader;
 
@@ -56,7 +61,7 @@ bool PbfProcessor::ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup
 		}
 
 		bool emitted = false;
-		if (!tags.empty() && nodeKeys.filter(tags)) {
+		if (output.canWriteNodes() && !tags.empty() && nodeKeys.filter(tags)) {
 			emitted = output.setNode(static_cast<NodeID>(nodeId), latplon, tags);
 		}
 
@@ -142,7 +147,7 @@ bool PbfProcessor::ReadWays(
 		if (llVec.empty()) continue;
 
 		try {
-			bool emitted = output.setWay(static_cast<WayID>(pbfWay.id), llVec, tags);
+			bool emitted = output.canWriteWays() && output.setWay(static_cast<WayID>(pbfWay.id), llVec, tags);
 
 			// If we need it for later, store the way's coordinates in the global way store
 			if (emitted || osmStore.way_is_used(wayId)) {
@@ -370,16 +375,27 @@ bool PbfProcessor::ReadBlock(
 		auto output_progress = [&]()
 		{
 			if (ioMutex.try_lock()) {
-				std::ostringstream str;
-				str << "\r";
-				void_mmap_allocator::reportStoreSize(str);
-				if (effectiveShards > 1)
-					str << std::to_string(shard + 1) << "/" << std::to_string(effectiveShards) << " ";
-
-				// TODO: revive showing the # of ways/relations?
-				str << "Block " << blocksProcessed.load() << "/" << blocksToProcess.load() << " ";
-				std::cout << str.str();
-				std::cout.flush();
+				// If we're interactive, show an update for each block.
+				// If we're not interactive, show an update for each 1% of blocks.
+				uint64_t blockProgress = blocksProcessed.load();
+				uint64_t minimumIncrement = blocksToProcess.load() / 100;
+				if (minimumIncrement < 1 || ISATTY)
+					minimumIncrement = 1;
+
+				if (phaseProgress == 0 || phaseProgress + minimumIncrement <= blockProgress) {
+					phaseProgress = blockProgress;
+
+					std::ostringstream str;
+					str << "\r";
+					void_mmap_allocator::reportStoreSize(str);
+					if (effectiveShards > 1)
+						str << std::to_string(shard + 1) << "/" << std::to_string(effectiveShards) << " ";
+
+					// TODO: revive showing the # of ways/relations?
+					str << "Block " << blocksProcessed.load() << "/" << blocksToProcess.load() << " ";
+					std::cout << str.str();
+					std::cout.flush();
+				}
 				ioMutex.unlock();
 			}
 		};
@@ -397,8 +413,13 @@ bool PbfProcessor::ReadBlock(
 			bool done = ScanWays(output, pg, pb, wayKeys);
 			if(done) { 
 				if (ioMutex.try_lock()) {
-					std::cout << "\r(Scanning for nodes used in ways: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
-					std::cout.flush();
+					size_t scanProgress = 100*blocksProcessed.load()/blocksToProcess.load();
+
+					if (scanProgress != phaseProgress) {
+						phaseProgress = scanProgress;
+						std::cout << "\r(Scanning for nodes used in ways: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
+						std::cout.flush();
+					}
 					ioMutex.unlock();
 				}
 				continue;
@@ -410,8 +431,13 @@ bool PbfProcessor::ReadBlock(
 			bool done = ScanRelations(output, pg, pb, wayKeys);
 			if(done) { 
 				if (ioMutex.try_lock()) {
-					std::cout << "\r(Scanning for ways used in relations: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
-					std::cout.flush();
+					size_t scanProgress = 100*blocksProcessed.load()/blocksToProcess.load();
+					
+					if (scanProgress != phaseProgress) {
+						phaseProgress = scanProgress;
+						std::cout << "\r(Scanning for ways used in relations: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%)           ";
+						std::cout.flush();
+					}
 					ioMutex.unlock();
 				}
 				continue;
@@ -591,6 +617,7 @@ int PbfProcessor::ReadPbfFile(
 	all_phases.push_back(ReadPhase::Relations);
 
 	for(auto phase: all_phases) {
+		phaseProgress = 0;
 		uint effectiveShards = 1;
 
 		// On memory-constrained machines, we might read ways/relations
diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp
@@ -345,7 +345,7 @@ int main(const int argc, const char* argv[]) {
 	std::mutex io_mutex;
 
 	// Loop through tiles
-	std::atomic<uint64_t> tilesWritten(0);
+	std::atomic<uint64_t> tilesWritten(0), lastTilesWritten(0);
 
 	for (auto source : sources) {
 		source->finalize(options.threadNum);
@@ -518,7 +518,7 @@ int main(const int argc, const char* argv[]) {
 			batchSize++;
 		}
 
-		boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten]() {
+		boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten, &lastTilesWritten]() {
 			std::vector<std::string> tileTimings;
 			std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize);
 			for(std::size_t i = startIndex; i < endIndex; ++i) {
@@ -557,16 +557,21 @@ int main(const int argc, const char* argv[]) {
 			tilesWritten += (endIndex - startIndex); 
 
 			if (io_mutex.try_lock()) {
-				// Show progress grouped by z6 (or lower)
-				size_t z = tileCoordinates[startIndex].first;
-				size_t x = tileCoordinates[startIndex].second.x;
-				size_t y = tileCoordinates[startIndex].second.y;
-				if (z > CLUSTER_ZOOM) {
-					x = x / (1 << (z - CLUSTER_ZOOM));
-					y = y / (1 << (z - CLUSTER_ZOOM));
-					z = CLUSTER_ZOOM;
+				uint64_t written = tilesWritten.load();
+
+				if (written >= lastTilesWritten + tileCoordinates.size() / 100 || ISATTY) {
+					lastTilesWritten = written;
+					// Show progress grouped by z6 (or lower)
+					size_t z = tileCoordinates[startIndex].first;
+					size_t x = tileCoordinates[startIndex].second.x;
+					size_t y = tileCoordinates[startIndex].second.y;
+					if (z > CLUSTER_ZOOM) {
+						x = x / (1 << (z - CLUSTER_ZOOM));
+						y = y / (1 << (z - CLUSTER_ZOOM));
+						z = CLUSTER_ZOOM;
+					}
+					cout << "z" << z << "/" << x << "/" << y << ", writing tile " << written << " of " << tileCoordinates.size() << "               \r" << std::flush;
 				}
-				cout << "z" << z << "/" << x << "/" << y << ", writing tile " << tilesWritten.load() << " of " << tileCoordinates.size() << "               \r" << std::flush;
 				io_mutex.unlock();
 			}
 		});
