ignfab · mborne · Jun 12, 2024 · Jun 12, 2024 · Jun 12, 2024
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # osm-integration
 
-**!!!EXPERIMENTAL!!!** Scripts [import.sh](import.sh) et [update.sh](update.sh) pour les chargements des données OSM dans PostgreSQL s'appuyant sur [osm2pgsql](https://osm2pgsql.org/doc/manual.html) et [osm2pgsql-replication](https://osm2pgsql.org/doc/man/osm2pgsql-replication-1.9.1.html).
+**!!!EXPERIMENTAL!!!** Scripts d'import et de mise à jour des données OSM dans PostgreSQL s'appuyant sur [osm2pgsql](https://osm2pgsql.org/doc/manual.html) et [osm2pgsql-replication](https://osm2pgsql.org/doc/man/osm2pgsql-replication-1.9.1.html).
 
 ## Prérequis
 
@@ -22,11 +22,12 @@ La connexion à la base de données s'appuie sur les variables d'environnements
 
 | Variable           | Description                                                        | Valeur par défaut                                          |
 | ------------------ | ------------------------------------------------------------------ | ---------------------------------------------------------- |
-| **OSM_PLANET_URL** | URL du fichier PBF pour [import.sh](import.sh)                     | https://download.geofabrik.de/europe/monaco-latest.osm.pbf |
+| **OSM_PLANET_URL** | URL du fichier PBF utilisée uniquement pour l'import               | https://download.geofabrik.de/europe/monaco-latest.osm.pbf |
 | OSM_DATA_DIR       | Dossier de téléchargement des données                              | `./data`                                                   |
 | CACHE_SIZE         | Permet d'adapter la taille du cache pour les noeuds                | `2000` (1)                                                 |
 | USE_FLAT_NODES     | Permet d'activer `--flat-nodes=${OSM_DATA_DIR}/nodes.raw`          | `0` (2)                                                    |
 | CREATE_DB          | Permet de désactiver la création automatique de la base de données | `0`                                                        |
+| LOG_PROGRESS       | Permet de désactiver le reporting de la progression                | `1`                                                        |
 
 Remarques :
 
@@ -62,7 +63,7 @@ export OSM_PLANET_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
 #export USE_FLAT_NODES=1
 #export OSM_PLANET_URL=https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf
 
-bash osm2pgsql/import.sh
+bash bin/import.sh
 ```
 
 ## Utlisation avec docker
@@ -79,17 +80,17 @@ docker compose up -d
 # Configurer l'import
 export OSM_PLANET_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
 export CACHE_SIZE=2000
-docker compose run terminal bin/update.sh
+docker compose run integration bin/update.sh
 
 # Mettre à jour les données
-docker compose run terminal bin/update.sh
+docker compose run integration bin/update.sh
 ```
 
 Pour le debug :
 
 ```bash
 # Se connecter à osm-integration en mode terminal
-docker compose exec terminal /bin/bash
+docker compose exec integration /bin/bash
 #... on est alors dans le conteneur :
 
 # Vérifier l'accès à la BDD

diff --git a/bin/import.sh b/bin/import.sh
@@ -1,38 +1,19 @@
 #!/bin/bash
 
-#--------------------------------------------------------------
-# Compute paths
-#--------------------------------------------------------------
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-ROOT_DIR=$(dirname "$SCRIPT_DIR")
 
-OSM_STYLES_DIR=${ROOT_DIR}/styles
-OSM_CARTO_DIR=${OSM_STYLES_DIR}/openstreetmap-carto
+source "${SCRIPT_DIR}/setenv.sh" || { 
+    exit 1 
+}
 
 #--------------------------------------------------------------
 # Handle import / update options
 #--------------------------------------------------------------
-OSM_DATA_DIR=${OSM_DATA_DIR:-${ROOT_DIR}/data}
-# optionnal
-OSM_FLAT_NODES_PATH=${OSM_DATA_DIR}/nodes.raw
 
 PGDATABASE=${PGDATABASE:-osm}
 echo "[INFO] PGDATABASE=${PGDATABASE}"
 
-# handle cache options
-CACHE_SIZE=${CACHE_SIZE:-2000}
-echo "[INFO] CACHE_SIZE=${CACHE_SIZE}"
-OPTS_CACHE="--cache=$CACHE_SIZE"
-
-USE_FLAT_NODES=${USE_FLAT_NODES:-0}
-echo "[INFO] USE_FLAT_NODES=${USE_FLAT_NODES}"
-if [ "$USE_FLAT_NODES" != "0" ];
-then
-    OPTS_CACHE="--flat-nodes ${OSM_FLAT_NODES_PATH} --cache=0"
-fi
-
-echo "[DEBUG] OPTS_CACHE=${OPTS_CACHE}"
+echo "[DEBUG] OSM2PGSQL_OPTS=${OSM2PGSQL_OPTS}"
 
 #--------------------------------------------------------------
 # Handle import specific options
@@ -44,26 +25,17 @@ echo "[INFO] OSM_PLANET_URL=${OSM_PLANET_URL}"
 FORCE_DOWNLOAD=${FORCE_DOWNLOAD:-0}
 echo "[INFO] FORCE_DOWNLOAD=${FORCE_DOWNLOAD}"
 
-#--------------------------------------------------------------
-# Create osm database
-#--------------------------------------------------------------
-createdb $PGDATABASE
 CREATE_DB=${CREATE_DB:-1}
 echo "[INFO] CREATE_DB=${CREATE_DB}"
 
 #--------------------------------------------------------------
 # Create osm database
 #--------------------------------------------------------------
-createdb $PGDATABASE
-psql -d $PGDATABASE -c "CREATE EXTENSION IF NOT EXISTS postgis"
-psql -d $PGDATABASE -c "CREATE EXTENSION IF NOT EXISTS hstore"
-
-#--------------------------------------------------------------
-# Download openstreetmap-carto if required
-#--------------------------------------------------------------
-if [ ! -e "${OSM_CARTO_DIR}/openstreetmap-carto.style" ];
+if [ "$CREATE_DB" != "0" ];
 then
-    bash "${SCRIPT_DIR}/get-styles.sh"
+    createdb "$PGDATABASE"
+    psql -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS postgis"
+    psql -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS hstore"
 fi
 
 #--------------------------------------------------------------
@@ -77,10 +49,10 @@ python3 "scripts/get-external-data.py" \
 #--------------------------------------------------------------
 # Download OSM data
 #--------------------------------------------------------------
-mkdir -p $OSM_DATA_DIR
+mkdir -p "$OSM_DATA_DIR"
 if [ ! -e "$OSM_DATA_DIR/osm.pbf" ] || [ "$FORCE_DOWNLOAD" != "0" ] ;
 then
-    wget -O "$OSM_DATA_DIR/osm.pbf" $OSM_PLANET_URL
+    wget -O "$OSM_DATA_DIR/osm.pbf" "$OSM_PLANET_URL"
 fi
 
 #--------------------------------------------------------------
@@ -94,11 +66,7 @@ rm -rf "${OSM_FLAT_NODES_PATH}"
 #--------------------------------------------------------------
 osm2pgsql \
     --database=$PGDATABASE \
-    --slim \
-    --hstore \
-    --multi-geometry \
-    --log-progress=true \
-    ${OPTS_CACHE} \
+    ${OSM2PGSQL_OPTS} \
     --style="${OSM_CARTO_DIR}/openstreetmap-carto.style" \
     --tag-transform-script="${OSM_CARTO_DIR}/openstreetmap-carto.lua" \
     "$OSM_DATA_DIR/osm.pbf"

diff --git a/bin/setenv.sh b/bin/setenv.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT_DIR=$(dirname "$SCRIPT_DIR")
+
+#--------------------------------------------------------------
+# Compute static paths
+#--------------------------------------------------------------
+export OSM_INTEGRATION_DIR=${ROOT_DIR}
+export OSM_STYLES_DIR=${ROOT_DIR}/styles
+export OSM_CARTO_DIR=${OSM_STYLES_DIR}/openstreetmap-carto
+
+#--------------------------------------------------------------
+# Download openstreetmap-carto if required
+#--------------------------------------------------------------
+if [ ! -e "${OSM_CARTO_DIR}/openstreetmap-carto.style" ];
+then
+    bash "${SCRIPT_DIR}/get-styles.sh"
+fi
+
+#--------------------------------------------------------------
+# Compute storage paths
+#--------------------------------------------------------------
+export OSM_DATA_DIR=${OSM_DATA_DIR:-${ROOT_DIR}/data}
+# optionnal (only for USE_FLAT_NODES)
+export OSM_FLAT_NODES_PATH=${OSM_DATA_DIR}/nodes.raw
+
+
+#--------------------------------------------------------------
+# static osm2pgsql options
+#--------------------------------------------------------------
+OSM2PGSQL_OPTS="--slim --hstore --multi-geometry"
+
+LOG_PROGRESS=${LOG_PROGRESS:-1}
+echo "[INFO] LOG_PROGRESS=${LOG_PROGRESS}"
+if [ "$LOG_PROGRESS" != "0" ];
+then
+    OSM2PGSQL_OPTS="${OSM2PGSQL_OPTS} --log-progress=true"
+else
+    OSM2PGSQL_OPTS="${OSM2PGSQL_OPTS} --log-progress=false"
+fi
+
+#--------------------------------------------------------------
+# Handle options for --flat-nodes and --cache
+#--------------------------------------------------------------
+
+USE_FLAT_NODES=${USE_FLAT_NODES:-0}
+echo "[INFO] USE_FLAT_NODES=${USE_FLAT_NODES}"
+if [ "$USE_FLAT_NODES" != "0" ];
+then
+    OSM2PGSQL_OPTS="--flat-nodes ${OSM_FLAT_NODES_PATH}"
+fi
+
+# Use CACHE_SIZE=0 with USE_FLAT_NODES 
+CACHE_SIZE=${CACHE_SIZE:-2000}
+echo "[INFO] CACHE_SIZE=${CACHE_SIZE}"
+OSM2PGSQL_OPTS="${OSM2PGSQL_OPTS} --cache=$CACHE_SIZE"
+
+#--------------------------------------------------------------
+# Compute storage paths
+#--------------------------------------------------------------
+export OSM2PGSQL_OPTS
diff --git a/bin/update.sh b/bin/update.sh
@@ -1,57 +1,26 @@
 #!/bin/bash
 
-#--------------------------------------------------------------
-# Compute paths
-#--------------------------------------------------------------
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-ROOT_DIR=$(dirname "$SCRIPT_DIR")
 
-OSM_STYLES_DIR=${ROOT_DIR}/styles
-OSM_CARTO_DIR=${OSM_STYLES_DIR}/openstreetmap-carto
+source "${SCRIPT_DIR}/setenv.sh" || { 
+    exit 1 
+}
 
 #--------------------------------------------------------------
 # Handle import / update options
 #--------------------------------------------------------------
-OSM_DATA_DIR=${OSM_DATA_DIR:-${ROOT_DIR}/data}
-# optionnal
-OSM_FLAT_NODES_PATH=${OSM_DATA_DIR}/nodes.raw
 
 PGDATABASE=${PGDATABASE:-osm}
 echo "[INFO] PGDATABASE=${PGDATABASE}"
 
-# handle cache options
-CACHE_SIZE=${CACHE_SIZE:-2000}
-echo "[INFO] CACHE_SIZE=${CACHE_SIZE}"
-OPTS_CACHE="--cache=$CACHE_SIZE"
-
-USE_FLAT_NODES=${USE_FLAT_NODES:-0}
-echo "[INFO] USE_FLAT_NODES=${USE_FLAT_NODES}"
-if [ "$USE_FLAT_NODES" != "0" ];
-then
-    OPTS_CACHE="--flat-nodes ${OSM_FLAT_NODES_PATH} --cache=0"
-fi
-
-echo "[INFO] OPTS_CACHE=${OPTS_CACHE}"
-
-#--------------------------------------------------------------
-# Download openstreetmap-carto if required
-#--------------------------------------------------------------
-if [ ! -e "${OSM_CARTO_DIR}/openstreetmap-carto.style" ];
-then
-    bash "${SCRIPT_DIR}/get-styles.sh"
-fi
+echo "[INFO] OSM2PGSQL_OPTS=${OSM2PGSQL_OPTS}"
 
 #--------------------------------------------------------------
 # Update OSM data in PostgreSQL
 #--------------------------------------------------------------
 osm2pgsql-replication update --verbose -- \
     --database=$PGDATABASE \
-    --slim \
-    --hstore \
-    --multi-geometry \
-    --log-progress=true \
-    ${OPTS_CACHE} \
+    ${OSM2PGSQL_OPTS} \
     --style="${OSM_CARTO_DIR}/openstreetmap-carto.style" \
     --tag-transform-script="${OSM_CARTO_DIR}/openstreetmap-carto.lua"
 
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,5 +1,5 @@
 services:
-  terminal:
+  integration:
     image: osm-integration
     container_name: osm_integration
     command: [ "/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ]