1.0.0

Dockerfile

@@ -0,0 +1,7 @@
+FROM sky_stareath_tiler_base:1.0.0
+
+COPY ./conf/uwsgi.ini /work/sky_stareath_tiler/conf/uwsgi.ini
+COPY ./src /work/sky_stareath_tiler/src
+COPY ./bin /work/sky_stareath_tiler/bin
+
+WORKDIR /work/sky_stareath_tiler/src

base/Dockerfile

@@ -1,10 +1,36 @@
-FROM debian11_py3.9_slim:latest
+FROM debian11_py3.9.13_gdal3.4.3_small:1.0.0
 
 COPY ./requirements.txt /tmp/
 COPY ./conf/bootstrap.sh /opt/bootstrap.sh
 COPY ./conf/supervisord.conf /etc/
 COPY ./conf/supervisor.conf /etc/supervisord.d/supervisor.conf
 
+RUN apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y  \
+               supervisor \
+               gcc g++  \
+               vim net-tools \
+               curl \
+               apt-utils \
+    && pip install --index-url https://pypi.tuna.tsinghua.edu.cn/simple --upgrade pip setuptools \
+    && pip install \
+        --index-url https://pypi.tuna.tsinghua.edu.cn/simple \
+        -r /tmp/requirements.txt \
+    # 默认是128,当server处理请求较慢，以至于socket监听队列被填满后，新来的请求会被拒绝。
+    && mkdir /opt/init  \
+    && echo "echo 2048 > /proc/sys/net/core/somaxconn" > /opt/init/sysctl.sh  \
+    && chmod +x /opt/bootstrap.sh  \
+    # 清理和删除工作
+    && apt-get autoremove -y  \
+    && apt-get autoclean -y \
+    && apt-get clean -y \
+    && rm -rf /tmp/* /var/tmp/*  \
+    && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete  \
+    && find /var/cache -type f -delete \
+    && find /var/log -type f | while read f; do echo -n '' > ${f}; done \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf ~/.cache/pip/*
 
 ENTRYPOINT ["/opt/bootstrap.sh"]
 

bin/worker.py

@@ -4,12 +4,12 @@ from multiprocessing import Process
 
 import rq
 
-from config import Config
-from connection import Connection
-
 src_root = os.path.join(os.path.abspath(__file__).split('bin')[0], 'src')
 sys.path.append(src_root)
 
+from config import Config
+from connection import Connection
+
 
 def start_worker(config):
     connection = Connection(config)
@@ -34,4 +34,4 @@ if __name__ == '__main__':
     # for p in process_list:
     #     p.join()
 
-    start_worker(config)
+    start_worker(config)

conf/uwsgi.ini

@@ -81,7 +81,7 @@ disable-logging = $(UWSGI_DISABLE_LOGGING)
 endif =
 
 # 记录uwsgi自身的日志
-logto = /data/logs/uwsgi.log
+logto = /data/logs/sky_stareath_tiler.log
 # daemonize = /data/logs/nginx_agent.log
 
 

config/config.yml

@@ -1,27 +1,29 @@
 common:
-  CACHE_PATH: /Users/mac/data/cache
-  OUTPUT_PATH:  /Users/mac/data/output
-  IMAGE_PATH: /Users/mac/data/image  # 影像数据目录
-  TILESET_PATH: /Users/mac/data/tileset  # 3Dtiles数据目录
+  CACHE_PATH: /data/cache  # 缓存目录
+  OUTPUT_PATH:  /data/output  # 输出目录
+  IMAGE_PATH: /data/image  # 影像数据目录
+  TILESET_PATH: /data/tileset  # 3Dtiles数据目录
   WORKER_NUM: 5
-  JOB_TIMEOUT: 3600
+  JOB_TIMEOUT: 36000
 
 dms:
-  URL: http://121.43.55.7:10081
+  URL: http://121.43.55.7:10081/dms
   columnId: 1364
   modelId: 1249
 
 oauth:
-  URL:
-  userName:
-  password:
-  clientId:
-  serviceId:
+  URL: http://121.43.55.7:10086/oauth
+  userName: user001
+  password: 1234567890
+  clientId: 1364
+  serviceId: 0
+
 nginx:
-  URL:
+  URL: http://121.43.55.7:80
+
 redis:
   # redis配置
-  HOST: 127.0.0.1 # 使用容器运行服务时,该HOST信息无用,容器内部会自动获取redis容器的ip
+  HOST: 192.168.1.18 # 使用容器运行服务时,该HOST信息无用,容器内部会自动获取redis容器的ip
   PORT: 6379 # 不要修改该端口号,除非明确了解使用场景
   DB: 0
 

docker-compose-v3.yml

@@ -0,0 +1,42 @@
+version: '3'
+services:
+  sky-gistools-server:
+      container_name: sky_stareath_tiler
+
+      image: sky_stareath_tiler:1.0.0
+
+      ports:
+        - "5000:5000"
+
+      environment:
+        - UWSGI_PROCESSES=8
+        - UWSGI_ENABLE_THREADS=true
+        - UWSGI_HTTP_PROCESSES=4
+        - UWSGI_HTTP_KEEPALIVE=false
+        - UWSGI_HARAKIRI=600
+        - UWSGI_TIMEOUT=600
+        - UWSGI_PY_OPTIMIZE=0
+
+      volumes:
+#        - /data/logs:/data/logs # 日志
+        - /Users/mac/data/tif:/data/tif  # 影像数据目录
+#        - /data/tileset:/data/tileset  # 3Dtiles数据目录
+        - /Users/mac/data/output:/data/output
+        - ./config:/work/sky_stareath_tiler/config  # 配置文件目录，包括软件配置信息
+
+      privileged: true
+
+      restart: always
+  redis:
+
+    image: docker.jcing.com/centos7_redis
+    container_name: sky_starearth_tiler_redis
+    ports:
+      - "6379:6379"
+
+    volumes:
+      - /data//redis:/data/
+
+    privileged: true
+
+    restart: always

docker-compose.yml

@@ -0,0 +1,40 @@
+service:
+  container_name: sky_stareath_tiler
+
+  image: sky_stareath_tiler:1.0.0
+
+  ports:
+    - "5000:5000"
+
+  environment:
+    - UWSGI_PROCESSES=8
+    - UWSGI_ENABLE_THREADS=true
+    - UWSGI_HTTP_PROCESSES=4
+    - UWSGI_HTTP_KEEPALIVE=false
+    - UWSGI_HARAKIRI=600
+    - UWSGI_TIMEOUT=600
+    - UWSGI_PY_OPTIMIZE=0
+
+  volumes:
+    - /data/logs:/data/logs # 日志
+    - /Users/mac/data/tif:/data/tif  # 影像数据目录
+    - /data/tileset:/data/tileset  # 3Dtiles数据目录
+    - /data/output:/data/output  # 切片输出目录
+    - ./config:/work/sky_stareath_tiler/config  # 配置文件目录，包括软件配置信息
+
+  privileged: true
+
+  restart: always
+
+redis:
+  image: docker.jcing.com/centos7_redis
+
+  container_name: slice_redis_test
+
+  ports:
+    - "6379:6379"
+
+  volumes:
+    - /data/redis:/data/
+
+  restart: always

requirements.txt

@@ -0,0 +1,17 @@
+flask
+flask-cors
+flask_restx
+numpy
+pillow
+rq
+uwsgi
+pyyaml
+requests
+rasterio
+shapely
+geojson
+piexif
+supermercado
+morecantile
+redis
+tilecloud

src/app/api/mission.py

@@ -1,3 +1,4 @@
+import json
 import os.path
 import traceback
 
@@ -14,6 +15,10 @@ config = Config()
 ns = Namespace('mission', description='任务管理 API接口')
 
 slice_mission_parser = reqparse.RequestParser(bundle_errors=True)
+slice_mission_parser.add_argument(name='title', type=str, location='form', required=False, help='dms 标题')
+slice_mission_parser.add_argument(name='content', type=str, location='form', required=False, help='dms 描述')
+slice_mission_parser.add_argument(name='c_name', type=str, location='form', required=False, help='dms 名称')
+
 slice_mission_parser.add_argument(name='data_path', type=str, location='form', required=False, help='数据目录')
 slice_mission_parser.add_argument(name='tile_size', type=int, location='form', required=False, help='瓦片大小')
 slice_mission_parser.add_argument(name='tile_format', type=str, location='form', required=False,
@@ -33,6 +38,12 @@ class MissionAPI(Resource):
         """添加任务"""
         try:
             form = request.form
+
+            # dms
+            title = form.get('title')
+            content = form.get('content')
+            c_name = form.get('c_name')
+
             data_path = form.get('data_path')
             tile_size = int(form.get('tile_size', 256))
             tile_grid = form.get('tile_grid', 'WebMercatorQuad')
@@ -43,6 +54,39 @@ class MissionAPI(Resource):
 
             if not data_path:
                 return jsonify(code=StatesCode.PARA_ERROR, message='输入路径错误')
+            # 获取token
+            login_data = {
+                'userName': config.oauth.userName,
+                'password': config.oauth.password,
+                'clientId': config.oauth.clientId,
+                'serviceId': config.oauth.serviceId
+            }
+            login_rep = requests.post(config.oauth.URL + '/api/user/login', data=login_data).json()
+
+            # 添加DMS
+            headers = {'token': login_rep.get('message'), 'Content-Type': 'application/x-www-form-urlencoded'}
+            dms_data = \
+                {
+                    'content': json.dumps({
+                        "title": title,
+                        "content": content,
+                        "c_tile_grid": tile_grid,
+                        "c_name": c_name,
+                        "c_tile_format": tile_format,
+                        "c_epsg": '4326',
+                        "c_url": config.nginx.URL + config.common.OUTPUT_PATH,
+                        "c_content": "0",
+                        "c_note": "0",
+                        "c_tile_size": tile_size,
+                        "c_zoom_min": min_zoom,
+                        "c_zoom_max": max_zoom,
+                        "c_auto_zoom": auto_zoom
+                    }),
+                    'modelId': config.dms.modelId,
+                    'columnId': config.dms.columnId
+                }
+            dms_rep = requests.post(config.dms.URL + '/content/addContent', headers=headers, data=dms_data).json()
+            dms_id = dms_rep['content']
 
             application = Application()
             application.new_mission(
@@ -52,42 +96,13 @@ class MissionAPI(Resource):
                 tile_format=tile_format,
                 auto_zoom=auto_zoom,
                 min_zoom=min_zoom,
-                max_zoom=max_zoom
+                max_zoom=max_zoom,
+                # dms
+                dms_id=dms_id
             )
 
-            # 添加至dem
-            try:
-                # 获取token
-                login_data = {
-                    'userName': config.oauth.userName,
-                    'password': config.oauth.password,
-                    'clientId': config.oauth.clientId,
-                    'serviceId': config.oauth.serviceId
-                }
-                login_rep = requests.post(config.oauth.URL + '/api/user/login', data=login_data).json()
-                headers = {'token': login_rep.get('message')}
-                dms_data = {
-                    "title": os.path.basename(data_path),
-                    "content": os.path.basename(data_path),
-                    "c_tile_grid": tile_grid,
-                    "c_name": os.path.basename(data_path),
-                    "c_tile_format": tile_format,
-                    "c_epsg": '4326',
-                    "c_url": config.nginx.URL + config.common.OUTPUT_PATH,
-                    "c_content": "0",
-                    "c_note": "0",
-                    "c_tile_size": tile_size,
-                    "c_zoom_min": min_zoom,
-                    "c_zoom_max": max_zoom,
-                    "c_auto_zoom": auto_zoom
-                }
-                dms_rep = requests.post(config.dms.URL + '/content/addContent', headers=headers, data=dms_data)
-
-            except Exception as err:
-                traceback.print_exc()
-                print_log('dms添加失败')
-
-            return {"code": StatesCode.SUCCESS, "message": "任务添加成功"}
+            return {"code": StatesCode.SUCCESS,
+                    "message": "开始切片预计需要%s分钟，请耐心等待" % str(len(os.listdir(data_path)) * 8)}
 
         except Exception as err:
 

src/app/mission_jobs/application.py

@@ -1,5 +1,3 @@
-import os
-
 import rq
 
 from app.mission_jobs.image_slice_job import image_slice_job
@@ -12,7 +10,7 @@ class Application:
     # def __init__(self, config):
     #     self._config = config
 
-    def new_mission(self, data_path,  tile_size, tile_grid, tile_format, auto_zoom, min_zoom, max_zoom):
+    def new_mission(self, data_path, tile_size, tile_grid, tile_format, auto_zoom, min_zoom, max_zoom, dms_id):
         # 影像切片
         config = Config()
         connection = Connection(config)
@@ -28,5 +26,8 @@ class Application:
                           "tile_format": tile_format,
                           "auto_zoom": auto_zoom,
                           "min_zoom": min_zoom,
-                          "max_zoom": max_zoom}
+                          "max_zoom": max_zoom,
+                          "dms_id": dms_id
+                          },
+                  job_timeout=int(config.common.JOB_TIMEOUT)
                   )

src/app/mission_jobs/image_slice.py

@@ -82,7 +82,7 @@ def slice_zxy(
     storage_osmzxy_obj.storage()
 
     # 删除临时目录
-    # if os.path.exists(os.path.join(str(tmp_tiles), base_name)):
-    #     shutil.rmtree(os.path.join(str(tmp_tiles), base_name))
+    if os.path.exists(os.path.join(str(tmp_tiles), base_name)):
+        shutil.rmtree(os.path.join(str(tmp_tiles), base_name))
 
     print_log("OSM_ZXY存储完毕..")

src/app/mission_jobs/image_slice_job.py

@@ -1,5 +1,6 @@
 import os
 
+import requests
 import rq
 
 from app.mission_jobs.image_slice import slice_zxy
@@ -7,23 +8,58 @@ from config import Config
 from connection import Connection
 
 
-def image_slice_job(data_path, output_path, tile_size, tile_grid, tile_format, auto_zoom, min_zoom, max_zoom):
+def image_slice_job(data_path, output_path, tile_size, tile_grid, tile_format, auto_zoom, min_zoom, max_zoom, dms_id):
     config = Config()
-    connection = Connection(config)
+    # connection = Connection(config)
+
+    # # 添加任务
+    # q = rq.Queue(name='default', connection=connection.redis_conn)
+    # for file_name in os.listdir(data_path):
+    #     input_file = os.path.join(data_path, file_name)
+    #     if os.path.splitext(file_name)[-1] == ".tif":
+    #         q.enqueue(slice_zxy,
+    #                   kwargs={"input_file": input_file,
+    #                           "output_path": output_path,
+    #                           "tile_size": tile_size,
+    #                           "tile_grid": tile_grid,
+    #                           "tile_format": tile_format,
+    #                           "auto_zoom": auto_zoom,
+    #                           "min_zoom": min_zoom,
+    #                           "max_zoom": max_zoom},
+    #                   job_timeout=int(config.common.JOB_TIMEOUT)
+    #                   )
 
-    # 添加任务
-    q = rq.Queue(name='default', connection=connection.redis_conn)
     for file_name in os.listdir(data_path):
         input_file = os.path.join(data_path, file_name)
         if os.path.splitext(file_name)[-1] == ".tif":
-            q.enqueue(slice_zxy,
-                      kwargs={"input_file": input_file,
-                              "output_path": output_path,
-                              "tile_size": tile_size,
-                              "tile_grid": tile_grid,
-                              "tile_format": tile_format,
-                              "auto_zoom": auto_zoom,
-                              "min_zoom": min_zoom,
-                              "max_zoom": max_zoom},
-                      job_timeout=int(config.common.JOB_TIMEOUT)
-                      )
+            slice_zxy(
+                input_file=input_file,
+                output_path=output_path,
+                tile_size=tile_size,
+                tile_grid=tile_grid,
+                tile_format=tile_format,
+                auto_zoom=auto_zoom,
+                min_zoom=min_zoom,
+                max_zoom=max_zoom,
+            )
+
+    # 获取token
+    login_data = {
+        'userName': config.oauth.userName,
+        'password': config.oauth.password,
+        'clientId': config.oauth.clientId,
+        'serviceId': config.oauth.serviceId
+    }
+
+    login_rep = requests.post(config.oauth.URL + '/api/user/login', data=login_data).json()
+
+    headers = {'token': login_rep.get('message')}
+
+    # 修改dms
+    data = {
+        'id': dms_id,
+        'columnId': config.dms.columnId,
+        'state': 1
+    }
+
+    requests.post(config.dms.URL + '/content/updateAudit', headers=headers, data=data)