longtsing/nmc_met_io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Update get_station_data function.

Browse Source
This commit is contained in:
NMC-DAVE 2019-07-03 22:07:05 +08:00
parent 755c8d0587
commit 2663565490
15 changed files with 3978 additions and 3957 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

208
.gitignore vendored
View File

@ -1,104 +1,104 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/

1348
LICENSE
View File

File diff suppressed because it is too large Load Diff

12
MANIFEST.in
View File

@ -1,6 +1,6 @@
include README.md
include LICENSE
recursive-exclude * __pycache__
recursive-exclude * *.pyc
recursive-exclude * *.pyo
recursive-exclude * *.orig
include README.md
include LICENSE
recursive-exclude * __pycache__
recursive-exclude * *.pyc
recursive-exclude * *.pyo
recursive-exclude * *.orig

84
README.md
View File

@ -1,43 +1,43 @@
# 气象数据读写及访问程序库
提供对MICAPS文件, 卫星云图, 天气雷达等数据的读写, 并访问CIMISS和MICAPS CASSANDRA数据库文件等.
Only Python 3 is supported.
## Dependencies
Other required packages:
- numpy
- scipy
- xarray
- pandas
- pyproj
- protobuf
- urllib3
- python-dateutil
## Install
Using the fellowing command to install packages:
```
pip install git+git://github.com/nmcdev/nmc_met_io.git
```
or download the package and install:
```
git clone --recursive https://github.com/nmcdev/nmc_met_io.git
cd nmc_met_io
python setup.py install
```
## 设置CIMISS和MICAPS服务器的地址及用户信息
在系统用户目录下("C:\Users\用户名"), 新建文本文件config_met_io.ini, 里面内容模板为:
```
[CIMISS]
DNS = xx.xx.xx.xx
USER_ID = xxxxxxxxx
PASSWORD = xxxxxxxx
[MICAPS]
GDS_IP = xx.xx.xx.xx
GDS_PORT = xxxx
```
# 气象数据读写及访问程序库
提供对MICAPS文件, 卫星云图, 天气雷达等数据的读写, 并访问CIMISS和MICAPS CASSANDRA数据库文件等.
Only Python 3 is supported.
## Dependencies
Other required packages:
- numpy
- scipy
- xarray
- pandas
- pyproj
- protobuf
- urllib3
- python-dateutil
## Install
Using the fellowing command to install packages:
```
pip install git+git://github.com/nmcdev/nmc_met_io.git
```
or download the package and install:
```
git clone --recursive https://github.com/nmcdev/nmc_met_io.git
cd nmc_met_io
python setup.py install
```
## 设置CIMISS和MICAPS服务器的地址及用户信息
在系统用户目录下("C:\Users\用户名"), 新建文本文件config_met_io.ini, 里面内容模板为:
```
[CIMISS]
DNS = xx.xx.xx.xx
USER_ID = xxxxxxxxx
PASSWORD = xxxxxxxx
[MICAPS]
GDS_IP = xx.xx.xx.xx
GDS_PORT = xxxx
```
这里xxxx用相应的地址, 接口和用户信息代替.

718
nmc_met_io/DataBlock_pb2.py
View File

@ -1,359 +1,359 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: DataBlock.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='DataBlock.proto',
package='',
syntax='proto3',
serialized_pb=_b('\n\x0f\x44\x61taBlock.proto\"E\n\x0cStringResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\"M\n\x0f\x42yteArrayResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x11\n\tbyteArray\x18\x03 \x01(\x0c\"h\n\x18StringAndByteArrayResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61taName\x18\x03 \x01(\t\x12\x11\n\tbyteArray\x18\x04 \x01(\x0c\"[\n\x0e\x46ileInfoResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x0e\n\x06isFile\x18\x03 \x01(\x08\x12\x10\n\x08\x66ileSize\x18\x04 \x01(\x03\"\x94\x01\n\tMapResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12,\n\tresultMap\x18\x03 \x03(\x0b\x32\x19.MapResult.ResultMapEntry\x1a\x30\n\x0eResultMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42)\n\x1c\x63n.gov.cma.cimiss.gds.serverB\tDataBlockb\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_STRINGRESULT = _descriptor.Descriptor(
name='StringResult',
full_name='StringResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='StringResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='StringResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='name', full_name='StringResult.name', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=19,
serialized_end=88,
)
_BYTEARRAYRESULT = _descriptor.Descriptor(
name='ByteArrayResult',
full_name='ByteArrayResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='ByteArrayResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='ByteArrayResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='byteArray', full_name='ByteArrayResult.byteArray', index=2,
number=3, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=90,
serialized_end=167,
)
_STRINGANDBYTEARRAYRESULT = _descriptor.Descriptor(
name='StringAndByteArrayResult',
full_name='StringAndByteArrayResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='StringAndByteArrayResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='StringAndByteArrayResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='dataName', full_name='StringAndByteArrayResult.dataName', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='byteArray', full_name='StringAndByteArrayResult.byteArray', index=3,
number=4, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=169,
serialized_end=273,
)
_FILEINFORESULT = _descriptor.Descriptor(
name='FileInfoResult',
full_name='FileInfoResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='FileInfoResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='FileInfoResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='isFile', full_name='FileInfoResult.isFile', index=2,
number=3, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='fileSize', full_name='FileInfoResult.fileSize', index=3,
number=4, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=275,
serialized_end=366,
)
_MAPRESULT_RESULTMAPENTRY = _descriptor.Descriptor(
name='ResultMapEntry',
full_name='MapResult.ResultMapEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='key', full_name='MapResult.ResultMapEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='value', full_name='MapResult.ResultMapEntry.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=469,
serialized_end=517,
)
_MAPRESULT = _descriptor.Descriptor(
name='MapResult',
full_name='MapResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='MapResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='MapResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='resultMap', full_name='MapResult.resultMap', index=2,
number=3, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[_MAPRESULT_RESULTMAPENTRY, ],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=369,
serialized_end=517,
)
_MAPRESULT_RESULTMAPENTRY.containing_type = _MAPRESULT
_MAPRESULT.fields_by_name['resultMap'].message_type = _MAPRESULT_RESULTMAPENTRY
DESCRIPTOR.message_types_by_name['StringResult'] = _STRINGRESULT
DESCRIPTOR.message_types_by_name['ByteArrayResult'] = _BYTEARRAYRESULT
DESCRIPTOR.message_types_by_name['StringAndByteArrayResult'] = _STRINGANDBYTEARRAYRESULT
DESCRIPTOR.message_types_by_name['FileInfoResult'] = _FILEINFORESULT
DESCRIPTOR.message_types_by_name['MapResult'] = _MAPRESULT
StringResult = _reflection.GeneratedProtocolMessageType('StringResult', (_message.Message,), dict(
DESCRIPTOR = _STRINGRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:StringResult)
))
_sym_db.RegisterMessage(StringResult)
ByteArrayResult = _reflection.GeneratedProtocolMessageType('ByteArrayResult', (_message.Message,), dict(
DESCRIPTOR = _BYTEARRAYRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:ByteArrayResult)
))
_sym_db.RegisterMessage(ByteArrayResult)
StringAndByteArrayResult = _reflection.GeneratedProtocolMessageType('StringAndByteArrayResult', (_message.Message,), dict(
DESCRIPTOR = _STRINGANDBYTEARRAYRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:StringAndByteArrayResult)
))
_sym_db.RegisterMessage(StringAndByteArrayResult)
FileInfoResult = _reflection.GeneratedProtocolMessageType('FileInfoResult', (_message.Message,), dict(
DESCRIPTOR = _FILEINFORESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:FileInfoResult)
))
_sym_db.RegisterMessage(FileInfoResult)
MapResult = _reflection.GeneratedProtocolMessageType('MapResult', (_message.Message,), dict(
ResultMapEntry = _reflection.GeneratedProtocolMessageType('ResultMapEntry', (_message.Message,), dict(
DESCRIPTOR = _MAPRESULT_RESULTMAPENTRY,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:MapResult.ResultMapEntry)
))
,
DESCRIPTOR = _MAPRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:MapResult)
))
_sym_db.RegisterMessage(MapResult)
_sym_db.RegisterMessage(MapResult.ResultMapEntry)
DESCRIPTOR.has_options = True
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\034cn.gov.cma.cimiss.gds.serverB\tDataBlock'))
_MAPRESULT_RESULTMAPENTRY.has_options = True
_MAPRESULT_RESULTMAPENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
# @@protoc_insertion_point(module_scope)
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: DataBlock.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='DataBlock.proto',
package='',
syntax='proto3',
serialized_pb=_b('\n\x0f\x44\x61taBlock.proto\"E\n\x0cStringResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\"M\n\x0f\x42yteArrayResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x11\n\tbyteArray\x18\x03 \x01(\x0c\"h\n\x18StringAndByteArrayResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61taName\x18\x03 \x01(\t\x12\x11\n\tbyteArray\x18\x04 \x01(\x0c\"[\n\x0e\x46ileInfoResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\x0e\n\x06isFile\x18\x03 \x01(\x08\x12\x10\n\x08\x66ileSize\x18\x04 \x01(\x03\"\x94\x01\n\tMapResult\x12\x11\n\terrorCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12,\n\tresultMap\x18\x03 \x03(\x0b\x32\x19.MapResult.ResultMapEntry\x1a\x30\n\x0eResultMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42)\n\x1c\x63n.gov.cma.cimiss.gds.serverB\tDataBlockb\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_STRINGRESULT = _descriptor.Descriptor(
name='StringResult',
full_name='StringResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='StringResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='StringResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='name', full_name='StringResult.name', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=19,
serialized_end=88,
)
_BYTEARRAYRESULT = _descriptor.Descriptor(
name='ByteArrayResult',
full_name='ByteArrayResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='ByteArrayResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='ByteArrayResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='byteArray', full_name='ByteArrayResult.byteArray', index=2,
number=3, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=90,
serialized_end=167,
)
_STRINGANDBYTEARRAYRESULT = _descriptor.Descriptor(
name='StringAndByteArrayResult',
full_name='StringAndByteArrayResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='StringAndByteArrayResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='StringAndByteArrayResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='dataName', full_name='StringAndByteArrayResult.dataName', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='byteArray', full_name='StringAndByteArrayResult.byteArray', index=3,
number=4, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=169,
serialized_end=273,
)
_FILEINFORESULT = _descriptor.Descriptor(
name='FileInfoResult',
full_name='FileInfoResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='FileInfoResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='FileInfoResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='isFile', full_name='FileInfoResult.isFile', index=2,
number=3, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='fileSize', full_name='FileInfoResult.fileSize', index=3,
number=4, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=275,
serialized_end=366,
)
_MAPRESULT_RESULTMAPENTRY = _descriptor.Descriptor(
name='ResultMapEntry',
full_name='MapResult.ResultMapEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='key', full_name='MapResult.ResultMapEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='value', full_name='MapResult.ResultMapEntry.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=469,
serialized_end=517,
)
_MAPRESULT = _descriptor.Descriptor(
name='MapResult',
full_name='MapResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='errorCode', full_name='MapResult.errorCode', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='errorMessage', full_name='MapResult.errorMessage', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='resultMap', full_name='MapResult.resultMap', index=2,
number=3, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[_MAPRESULT_RESULTMAPENTRY, ],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=369,
serialized_end=517,
)
_MAPRESULT_RESULTMAPENTRY.containing_type = _MAPRESULT
_MAPRESULT.fields_by_name['resultMap'].message_type = _MAPRESULT_RESULTMAPENTRY
DESCRIPTOR.message_types_by_name['StringResult'] = _STRINGRESULT
DESCRIPTOR.message_types_by_name['ByteArrayResult'] = _BYTEARRAYRESULT
DESCRIPTOR.message_types_by_name['StringAndByteArrayResult'] = _STRINGANDBYTEARRAYRESULT
DESCRIPTOR.message_types_by_name['FileInfoResult'] = _FILEINFORESULT
DESCRIPTOR.message_types_by_name['MapResult'] = _MAPRESULT
StringResult = _reflection.GeneratedProtocolMessageType('StringResult', (_message.Message,), dict(
DESCRIPTOR = _STRINGRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:StringResult)
))
_sym_db.RegisterMessage(StringResult)
ByteArrayResult = _reflection.GeneratedProtocolMessageType('ByteArrayResult', (_message.Message,), dict(
DESCRIPTOR = _BYTEARRAYRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:ByteArrayResult)
))
_sym_db.RegisterMessage(ByteArrayResult)
StringAndByteArrayResult = _reflection.GeneratedProtocolMessageType('StringAndByteArrayResult', (_message.Message,), dict(
DESCRIPTOR = _STRINGANDBYTEARRAYRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:StringAndByteArrayResult)
))
_sym_db.RegisterMessage(StringAndByteArrayResult)
FileInfoResult = _reflection.GeneratedProtocolMessageType('FileInfoResult', (_message.Message,), dict(
DESCRIPTOR = _FILEINFORESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:FileInfoResult)
))
_sym_db.RegisterMessage(FileInfoResult)
MapResult = _reflection.GeneratedProtocolMessageType('MapResult', (_message.Message,), dict(
ResultMapEntry = _reflection.GeneratedProtocolMessageType('ResultMapEntry', (_message.Message,), dict(
DESCRIPTOR = _MAPRESULT_RESULTMAPENTRY,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:MapResult.ResultMapEntry)
))
,
DESCRIPTOR = _MAPRESULT,
__module__ = 'DataBlock_pb2'
# @@protoc_insertion_point(class_scope:MapResult)
))
_sym_db.RegisterMessage(MapResult)
_sym_db.RegisterMessage(MapResult.ResultMapEntry)
DESCRIPTOR.has_options = True
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\034cn.gov.cma.cimiss.gds.serverB\tDataBlock'))
_MAPRESULT_RESULTMAPENTRY.has_options = True
_MAPRESULT_RESULTMAPENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
# @@protoc_insertion_point(module_scope)

14
nmc_met_io/__init__.py
View File

@ -1,7 +1,7 @@
"""
dk_Met_IO is a package to read or retrieve meteorological data
from various data sources.
"""
__author__ = "The R & D Center for Weather Forecasting Technology in NMC, CMA"
__version__ = '0.1.0'
"""
dk_Met_IO is a package to read or retrieve meteorological data
from various data sources.
"""
__author__ = "The R & D Center for Weather Forecasting Technology in NMC, CMA"
__version__ = '0.1.0'

62
nmc_met_io/config.py
View File

@ -1,31 +1,31 @@
# _*_ coding: utf-8 _*_
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read configure file.
"""
import os
import configparser
def ConfigFetchError(Exception):
pass
def _get_config_from_rcfile():
"""
Get configure information from config_dk_met_io.ini file.
"""
rc = os.path.normpath(os.path.expanduser("~/config_met_io.ini"))
try:
config = configparser.ConfigParser()
config.read(rc)
except IOError as e:
raise ConfigFetchError(str(e))
except Exception as e:
raise ConfigFetchError(str(e))
return config
# _*_ coding: utf-8 _*_
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read configure file.
"""
import os
import configparser
def ConfigFetchError(Exception):
pass
def _get_config_from_rcfile():
"""
Get configure information from config_dk_met_io.ini file.
"""
rc = os.path.normpath(os.path.expanduser("~/config_met_io.ini"))
try:
config = configparser.ConfigParser()
config.read(rc)
except IOError as e:
raise ConfigFetchError(str(e))
except Exception as e:
raise ConfigFetchError(str(e))
return config

132
nmc_met_io/read_grads.py
View File

@ -1,66 +1,66 @@
# _*_ coding: utf-8 _*_
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read grads data file.
"""
import os
import re
from datetime import datetime
import numpy as np
def read_cmp_pre_hour_grid(files, start_lon=70.05, start_lat=15.05):
"""
Read SURF_CLI_CHN_MERGE_CMP_PRE_HOUR_GRID_0.1 data file
:param files: a single or multiple data filenames.
:param start_lon: region lower left corner longitude.
:param start_lat: region lower left corner latitude.
:return: data and time
:Examples:
>>> files = ("F:/201607/SURF_CLI_CHN_MERGE_CMP_"
"PRE_HOUR_GRID_0.10-2016070100.grd")
>>> data, time, lon, lat = read_cmp_pre_hour_grid(files)
"""
# sort and count data files
if isinstance(files, str):
files = np.array([files])
else:
files = np.array(files)
# define coordinates
lon = np.arange(700) * 0.1 + start_lon
lat = np.arange(440) * 0.1 + start_lat
# define variables
data = np.full((len(files), lat.size, lon.size), np.nan)
time = []
# loop every data file
for i, f in enumerate(files):
# check file exist
if not os.path.isfile(f):
return None, None, None, None
# extract time information
ttime = re.search('\d{10}', os.path.basename(f))
time.append(datetime.strptime(ttime.group(0), "%Y%m%d%H"))
# read data
try:
tdata = np.fromfile(
f, dtype=np.dtype('float32')).reshape(2, len(lat), len(lon))
tdata[tdata == -999.0] = np.nan # missing value
data[i, :, :] = tdata[0, :, :]
except IOError:
print("Can not read data from "+f)
continue
# return value
return data, time, lon, lat
# _*_ coding: utf-8 _*_
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read grads data file.
"""
import os
import re
from datetime import datetime
import numpy as np
def read_cmp_pre_hour_grid(files, start_lon=70.05, start_lat=15.05):
"""
Read SURF_CLI_CHN_MERGE_CMP_PRE_HOUR_GRID_0.1 data file
:param files: a single or multiple data filenames.
:param start_lon: region lower left corner longitude.
:param start_lat: region lower left corner latitude.
:return: data and time
:Examples:
>>> files = ("F:/201607/SURF_CLI_CHN_MERGE_CMP_"
"PRE_HOUR_GRID_0.10-2016070100.grd")
>>> data, time, lon, lat = read_cmp_pre_hour_grid(files)
"""
# sort and count data files
if isinstance(files, str):
files = np.array([files])
else:
files = np.array(files)
# define coordinates
lon = np.arange(700) * 0.1 + start_lon
lat = np.arange(440) * 0.1 + start_lat
# define variables
data = np.full((len(files), lat.size, lon.size), np.nan)
time = []
# loop every data file
for i, f in enumerate(files):
# check file exist
if not os.path.isfile(f):
return None, None, None, None
# extract time information
ttime = re.search('\d{10}', os.path.basename(f))
time.append(datetime.strptime(ttime.group(0), "%Y%m%d%H"))
# read data
try:
tdata = np.fromfile(
f, dtype=np.dtype('float32')).reshape(2, len(lat), len(lon))
tdata[tdata == -999.0] = np.nan # missing value
data[i, :, :] = tdata[0, :, :]
except IOError:
print("Can not read data from "+f)
continue
# return value
return data, time, lon, lat

1874
nmc_met_io/read_micaps.py
View File

File diff suppressed because it is too large Load Diff

228
nmc_met_io/read_radar.py
View File

@ -1,114 +1,114 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read CMA radar format file.
"""
from array import array
import numpy as np
def read_cma_sa_radar(fname):
"""
Read CMA SA radar format file.
refer to
https://github.com/smft/CMA_SA_RADAR_READER/blob/master/read_RADAR.py
https://github.com/smft/CMA_SA_RADAR_READER/blob/master/barb_plot.py
for plot map.
:param fname: file path name.
:return: data dictionary.
"""
# read data
flag = open(fname, "r")
data = np.asarray(array("B", flag.read()))
data = data.reshape([len(data)/2432, 2432])
# find elevation angle
if data[0, 72] == 11:
phi = [0.50, 0.50, 1.45, 1.45, 2.40, 3.35, 4.30,
5.25, 6.2, 7.5, 8.7, 10, 12, 14, 16.7, 19.5]
if data[0, 72] == 21:
phi = [0.50, 0.50, 1.45, 1.45, 2.40, 3.35, 4.30,
6.00, 9.00, 14.6, 19.5]
if data[0, 72] == 31:
phi = [0.50, 0.50, 1.50, 1.50, 2.50, 2.50, 3.50, 4.50]
if data[0, 72] == 32:
phi = [0.50, 0.50, 2.50, 3.50, 4.50]
# define data
g1 = np.zeros([len(data), 460])
h1 = np.zeros([len(data), 460])
i1 = np.zeros([len(data), 460])
j1 = np.zeros([len(data), 460])
# process data
count = 0
while count < len(data):
print("径向数据编号 : ", count)
b1 = data[count, 44] + 256 * data[count, 45] # 仰角序数
c1 = ((data[count, 36] + 256 * data[count, 37]) /
8 * 180 / 4096) # 方位角
d1 = data[count, 54] + 256 * data[count, 55] # 径向库
print("仰角序数,方位角,径向库 : ", b1, c1, d1)
if d1 == 0:
count += 1
continue
else:
count += 1
i = 0
while i < 460:
g1[count - 1, i] = phi[b1 - 1] # 仰角
h1[count - 1, i] = c1 # 方位角
i1[count - 1, i] = 0.5 + i - 1 # 径向
if i > d1: # 反射率
j1[count - 1, i] = 0
else:
if data[count - 1, 128 + i] == 0: # 无数据
j1[count - 1, i] = 0
else:
if data[count - 1, 128 + i] == 1: # 距离模糊
j1[count - 1, i] = 0
else: # 数据正常
j1[count - 1, i] = ((data[count - 1, 128 + i] - 2) /
2 - 32)
i += 1
# calculate angle index
n = 3
a2 = 0 # 仰角序数
while a2 < len(data):
if data[a2, 44] > (n - 1):
break
a2 += 1
a3 = a2
while a3 < len(data):
if data[a3, 44] > n:
break
a3 += 1
# put data
yj = g1[a2:a3, :] # 仰角
fwj = h1[a2:a3, :] # 方位角
jx = i1[a2:a3, :] # 径向
fsl = j1[a2:a3, :] # 反射率
# return data
return yj, fwj, jx, fsl
def sph2cart(elevation, azimuth, r):
"""
Convert spherical coordinates to cartesian.
"""
ele, a = np.deg2rad([elevation, azimuth])
x = r * np.cos(ele) * np.cos(a)
y = r * np.cos(ele) * np.sin(a)
z = r * np.sin(ele)
return x, y, z
# -*- coding: utf-8 -*-
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read CMA radar format file.
"""
from array import array
import numpy as np
def read_cma_sa_radar(fname):
"""
Read CMA SA radar format file.
refer to
https://github.com/smft/CMA_SA_RADAR_READER/blob/master/read_RADAR.py
https://github.com/smft/CMA_SA_RADAR_READER/blob/master/barb_plot.py
for plot map.
:param fname: file path name.
:return: data dictionary.
"""
# read data
flag = open(fname, "r")
data = np.asarray(array("B", flag.read()))
data = data.reshape([len(data)/2432, 2432])
# find elevation angle
if data[0, 72] == 11:
phi = [0.50, 0.50, 1.45, 1.45, 2.40, 3.35, 4.30,
5.25, 6.2, 7.5, 8.7, 10, 12, 14, 16.7, 19.5]
if data[0, 72] == 21:
phi = [0.50, 0.50, 1.45, 1.45, 2.40, 3.35, 4.30,
6.00, 9.00, 14.6, 19.5]
if data[0, 72] == 31:
phi = [0.50, 0.50, 1.50, 1.50, 2.50, 2.50, 3.50, 4.50]
if data[0, 72] == 32:
phi = [0.50, 0.50, 2.50, 3.50, 4.50]
# define data
g1 = np.zeros([len(data), 460])
h1 = np.zeros([len(data), 460])
i1 = np.zeros([len(data), 460])
j1 = np.zeros([len(data), 460])
# process data
count = 0
while count < len(data):
print("径向数据编号 : ", count)
b1 = data[count, 44] + 256 * data[count, 45] # 仰角序数
c1 = ((data[count, 36] + 256 * data[count, 37]) /
8 * 180 / 4096) # 方位角
d1 = data[count, 54] + 256 * data[count, 55] # 径向库
print("仰角序数,方位角,径向库 : ", b1, c1, d1)
if d1 == 0:
count += 1
continue
else:
count += 1
i = 0
while i < 460:
g1[count - 1, i] = phi[b1 - 1] # 仰角
h1[count - 1, i] = c1 # 方位角
i1[count - 1, i] = 0.5 + i - 1 # 径向
if i > d1: # 反射率
j1[count - 1, i] = 0
else:
if data[count - 1, 128 + i] == 0: # 无数据
j1[count - 1, i] = 0
else:
if data[count - 1, 128 + i] == 1: # 距离模糊
j1[count - 1, i] = 0
else: # 数据正常
j1[count - 1, i] = ((data[count - 1, 128 + i] - 2) /
2 - 32)
i += 1
# calculate angle index
n = 3
a2 = 0 # 仰角序数
while a2 < len(data):
if data[a2, 44] > (n - 1):
break
a2 += 1
a3 = a2
while a3 < len(data):
if data[a3, 44] > n:
break
a3 += 1
# put data
yj = g1[a2:a3, :] # 仰角
fwj = h1[a2:a3, :] # 方位角
jx = i1[a2:a3, :] # 径向
fsl = j1[a2:a3, :] # 反射率
# return data
return yj, fwj, jx, fsl
def sph2cart(elevation, azimuth, r):
"""
Convert spherical coordinates to cartesian.
"""
ele, a = np.deg2rad([elevation, azimuth])
x = r * np.cos(ele) * np.cos(a)
y = r * np.cos(ele) * np.sin(a)
z = r * np.sin(ele)
return x, y, z

358
nmc_met_io/read_satellite.py
View File

@ -1,179 +1,179 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read FY satellite awx format file.
"""
import numpy as np
def read_awx_cloud(fname):
"""
Read satellite awx format file.
:param fname: file pathname.
:return: data list
"""
# read part of binary
# refer to
# https://stackoverflow.com/questions/14245094/how-to-read-part-of-binary-file-with-numpy
pass
def read_himawari(fname, resolution):
"""
Read the japan himawari satellite standard data file.
refere to
https://github.com/smft/Read_Himawari_binary_data/blob/master/read_Himawari.py
:param fname: data file pathname.
:param resolution: data resolution.
:return: data list.
"""
# define resolution
if resolution == 1:
res = 12100000
nlin = 1100
ncol = 11000
elif resolution == 2:
res = 3025000
nlin = 550
ncol = 5500
else:
res = 48400000
nlin = 2200
ncol = 22000
# define binary file format
formation = [('bn', 'i1', 1),
('bl', 'i2', 1),
('thb', 'i2', 1),
('bo', 'i1', 1),
('sn', 'S1', 16),
('pcn', 'S1', 16),
('oa', 'S1', 4),
('obf', 'S1', 2),
('ot', 'i2', 1),
('ost', 'float64', 1),
('oet', 'float64', 1),
('fct', 'float64', 1),
('thl', 'i4', 1),
('tdl', 'i4', 1),
('qf1', 'i1', 1),
('qf2', 'i1', 1),
('qf3', 'i1', 1),
('qf4', 'i1', 1),
('ffv', 'S1', 32),
('fn', 'S1', 128),
('null1', 'S1', 40),
('bn2', 'i1', 1),
('bl2', 'i2', 1),
('nbpp', 'i2', 1),
('noc', 'i2', 1),
('nol', 'i2', 1),
('cffdb', 'i1', 1),
('null2', 'S1', 40),
('bn3', 'i1', 1),
('bl3', 'i2', 1),
('sl', 'float64', 1),
('CFAC', 'i4', 1),
('LFAC', 'i4', 1),
('COFF', 'float32', 1),
('LOFF', 'float32', 1),
('dfectvs', 'float64', 1),
('eer', 'float64', 1),
('epr', 'float64', 1),
('var1', 'float64', 1),
('var2', 'float64', 1),
('var3', 'float64', 1),
('cfsd', 'float64', 1),
('rt', 'i2', 1),
('rs', 'i2', 1),
('null3', 'S1', 40),
('bn4', 'i1', 1),
('bl4', 'i2', 1),
('ni', 'float64', 1),
('ssplon', 'float64', 1),
('ssplat', 'float64', 1),
('dfects4', 'float64', 1),
('nlat', 'float64', 1),
('nlon', 'float64', 1),
('sp', 'float64', 3),
('mp', 'float64', 3),
('null4', 'S1', 40),
('bn5', 'i1', 1),
('bl5', 'i2', 1),
('bdn', 'i2', 1),
('cwl', 'float64', 1),
('vnobpp', 'i2', 1),
('cvoep', 'uint16', 1),
('cvoposa', 'uint16', 1),
('gfcce', 'float64', 1),
('cfcce', 'float64', 1),
('c0', 'float64', 1),
('c1', 'float64', 1),
('c2', 'float64', 1),
('C0', 'float64', 1),
('C1', 'float64', 1),
('C2', 'float64', 1),
('sol', 'float64', 1),
('pc', 'float64', 1),
('bc', 'float64', 1),
('null5', 'S1', 40),
('b06n01', 'i1', 1),
('b06n02', 'i2', 1),
('b06n03', 'float64', 1),
('b06n04', 'float64', 1),
('b06n05', 'float64', 1),
('b06n06', 'float64', 1),
('b06n07', 'float64', 1),
('b06n08', 'float64', 1),
('b06n09', 'float64', 1),
('b06n10', 'float64', 1),
('b06n11', 'float32', 1),
('b06n12', 'float32', 1),
('b06n13', 'S1', 128),
('b06n14', 'S1', 56),
('b07n01', 'i1', 1),
('b07n02', 'i2', 1),
('b07n03', 'i1', 1),
('b07n04', 'i1', 1),
('b07n05', 'i2', 1),
('b07n06', 'S1', 40),
('b08n01', 'i1', 1),
('b08n02', 'i2', 1),
('b08n03', 'float32', 1),
('b08n04', 'float32', 1),
('b08n05', 'float64', 1),
('b08n06', 'i2', 1),
('b08n07', 'i2', 1),
('b08n08', 'float32', 1),
('b08n09', 'float32', 1),
('b08n10', 'S1', 50),
('b09n01', 'i1', 1),
('b09n02', 'i2', 1),
('b09n03', 'i2', 1),
('b09n04', 'i2', 1),
('b09n05', 'float64', 1),
('b09n06', 'S1', 70),
('b10n01', 'i1', 1),
('b10n02', 'i4', 1),
('b10n03', 'i2', 1),
('b10n04', 'i2', 1),
('b10n05', 'i2', 1),
('b10n06', 'S1', 36),
('b11n01', 'i1', 1),
('b11n02', 'i2', 1),
('b11n03', 'S1', 256),
('b12n01', 'i2', res)]
data = np.fromfile(fname, dtype=formation)['b12n01'].reshape(nlin, ncol)
return list(data)
# -*- coding: utf-8 -*-
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Read FY satellite awx format file.
"""
import numpy as np
def read_awx_cloud(fname):
"""
Read satellite awx format file.
:param fname: file pathname.
:return: data list
"""
# read part of binary
# refer to
# https://stackoverflow.com/questions/14245094/how-to-read-part-of-binary-file-with-numpy
pass
def read_himawari(fname, resolution):
"""
Read the japan himawari satellite standard data file.
refere to
https://github.com/smft/Read_Himawari_binary_data/blob/master/read_Himawari.py
:param fname: data file pathname.
:param resolution: data resolution.
:return: data list.
"""
# define resolution
if resolution == 1:
res = 12100000
nlin = 1100
ncol = 11000
elif resolution == 2:
res = 3025000
nlin = 550
ncol = 5500
else:
res = 48400000
nlin = 2200
ncol = 22000
# define binary file format
formation = [('bn', 'i1', 1),
('bl', 'i2', 1),
('thb', 'i2', 1),
('bo', 'i1', 1),
('sn', 'S1', 16),
('pcn', 'S1', 16),
('oa', 'S1', 4),
('obf', 'S1', 2),
('ot', 'i2', 1),
('ost', 'float64', 1),
('oet', 'float64', 1),
('fct', 'float64', 1),
('thl', 'i4', 1),
('tdl', 'i4', 1),
('qf1', 'i1', 1),
('qf2', 'i1', 1),
('qf3', 'i1', 1),
('qf4', 'i1', 1),
('ffv', 'S1', 32),
('fn', 'S1', 128),
('null1', 'S1', 40),
('bn2', 'i1', 1),
('bl2', 'i2', 1),
('nbpp', 'i2', 1),
('noc', 'i2', 1),
('nol', 'i2', 1),
('cffdb', 'i1', 1),
('null2', 'S1', 40),
('bn3', 'i1', 1),
('bl3', 'i2', 1),
('sl', 'float64', 1),
('CFAC', 'i4', 1),
('LFAC', 'i4', 1),
('COFF', 'float32', 1),
('LOFF', 'float32', 1),
('dfectvs', 'float64', 1),
('eer', 'float64', 1),
('epr', 'float64', 1),
('var1', 'float64', 1),
('var2', 'float64', 1),
('var3', 'float64', 1),
('cfsd', 'float64', 1),
('rt', 'i2', 1),
('rs', 'i2', 1),
('null3', 'S1', 40),
('bn4', 'i1', 1),
('bl4', 'i2', 1),
('ni', 'float64', 1),
('ssplon', 'float64', 1),
('ssplat', 'float64', 1),
('dfects4', 'float64', 1),
('nlat', 'float64', 1),
('nlon', 'float64', 1),
('sp', 'float64', 3),
('mp', 'float64', 3),
('null4', 'S1', 40),
('bn5', 'i1', 1),
('bl5', 'i2', 1),
('bdn', 'i2', 1),
('cwl', 'float64', 1),
('vnobpp', 'i2', 1),
('cvoep', 'uint16', 1),
('cvoposa', 'uint16', 1),
('gfcce', 'float64', 1),
('cfcce', 'float64', 1),
('c0', 'float64', 1),
('c1', 'float64', 1),
('c2', 'float64', 1),
('C0', 'float64', 1),
('C1', 'float64', 1),
('C2', 'float64', 1),
('sol', 'float64', 1),
('pc', 'float64', 1),
('bc', 'float64', 1),
('null5', 'S1', 40),
('b06n01', 'i1', 1),
('b06n02', 'i2', 1),
('b06n03', 'float64', 1),
('b06n04', 'float64', 1),
('b06n05', 'float64', 1),
('b06n06', 'float64', 1),
('b06n07', 'float64', 1),
('b06n08', 'float64', 1),
('b06n09', 'float64', 1),
('b06n10', 'float64', 1),
('b06n11', 'float32', 1),
('b06n12', 'float32', 1),
('b06n13', 'S1', 128),
('b06n14', 'S1', 56),
('b07n01', 'i1', 1),
('b07n02', 'i2', 1),
('b07n03', 'i1', 1),
('b07n04', 'i1', 1),
('b07n05', 'i2', 1),
('b07n06', 'S1', 40),
('b08n01', 'i1', 1),
('b08n02', 'i2', 1),
('b08n03', 'float32', 1),
('b08n04', 'float32', 1),
('b08n05', 'float64', 1),
('b08n06', 'i2', 1),
('b08n07', 'i2', 1),
('b08n08', 'float32', 1),
('b08n09', 'float32', 1),
('b08n10', 'S1', 50),
('b09n01', 'i1', 1),
('b09n02', 'i2', 1),
('b09n03', 'i2', 1),
('b09n04', 'i2', 1),
('b09n05', 'float64', 1),
('b09n06', 'S1', 70),
('b10n01', 'i1', 1),
('b10n02', 'i4', 1),
('b10n03', 'i2', 1),
('b10n04', 'i2', 1),
('b10n05', 'i2', 1),
('b10n06', 'S1', 36),
('b11n01', 'i1', 1),
('b11n02', 'i2', 1),
('b11n03', 'S1', 256),
('b12n01', 'i2', res)]
data = np.fromfile(fname, dtype=formation)['b12n01'].reshape(nlin, ncol)
return list(data)

332
nmc_met_io/retrieve_cimiss_history.py
View File

@ -1,166 +1,166 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Retrieve historical data from CIMISS service.
"""
import os
import calendar
import urllib.request
import numpy as np
from nmc_met_io.retrieve_cimiss_server import cimiss_obs_by_time_range
from nmc_met_io.retrieve_cimiss_server import cimiss_obs_in_rect_by_time_range
from nmc_met_io.retrieve_cimiss_server import cimiss_obs_file_by_time_range
def get_day_hist_obs(years=np.arange(2000, 2011, 1),
month_range=(1, 12),
elements=None,
sta_levels=None,
outfname='day_rain_obs',
outdir='.'):
"""
Download historical daily observations and write to data files,
each month a file.
:param years: years for historical data
:param month_range: month range each year, like (1, 12)
:param elements: elements for retrieve, 'ele1, ele2, ...'
:param sta_levels: station levels
:param outfname: output file name + '_year' + '_month'
:param outdir: output file directory
:return: output file names.
:Example:
>>> get_day_hist_obs(years=np.arange(2000, 2016, 1), outdir="D:/")
"""
# check elements
if elements is None:
elements = "Station_Id_C,Station_Name,Datetime,Lat,Lon,PRE_Time_0808"
# check output directory
if not os.path.exists(outdir):
os.makedirs(outdir)
# define months
months = np.arange(1, 13, 1)
# Because of the CIMISS data mount limit,
# so loop every year to download the data.
out_files = []
for iy in years:
if calendar.isleap(iy):
last_day = ['31', '29', '31', '30', '31', '30',
'31', '31', '30', '31', '30', '31']
else:
last_day = ['31', '28', '31', '30', '31', '30',
'31', '31', '30', '31', '30', '31']
for i, im in enumerate(months):
# check month range
if not (month_range[0] <= im <= month_range[1]):
continue
month = '%02d' % im
start_time = str(iy) + month + '01' + '000000'
end_time = str(iy) + month + last_day[i] + '000000'
time_range = "[" + start_time + "," + end_time + "]"
# retrieve observations from CIMISS server
data = cimiss_obs_by_time_range(
time_range, sta_levels=sta_levels,
data_code="SURF_CHN_MUL_DAY", elements=elements)
if data is None:
continue
# save observation data to file
out_files.append(os.path.join(
outdir, outfname + "_" + str(iy) + "_" + month + ".pkl"))
data.to_pickle(out_files[-1])
return out_files
def get_mon_hist_obs(years=np.arange(2000, 2011, 1),
limit=(3, 73, 54, 136),
elements=None,
outfname='mon_surface_obs',
outdir='.'):
"""
Download historical monthly observations and write to data files,
each year a file.
:param years: years for historical data
:param limit: spatial limit [min_lat, min_lon, max_lat, max_lon]
:param elements: elements for retrieve, 'ele1, ele2, ...'
:param outfname: output file name + 'year'
:param outdir: output file directory
:return: Output filenames
"""
# check elements
if elements is None:
elements = ("Station_Id_C,Station_Name,Year,"
"Mon,Lat,Lon,Alti,PRE_Time_0808")
# check output directory
if not os.path.exists(outdir):
os.makedirs(outdir)
# Loop every year to download the data.
out_files = []
for iy in years:
# check out file
outfile = os.path.join(outdir, outfname + "_" + str(iy) + ".pkl")
if os.path.isfile(outfile):
continue
# set time range
start_time = str(iy) + '0101' + '000000'
end_time = str(iy) + '1201' + '000000'
time_range = "[" + start_time + "," + end_time + "]"
# retrieve observations from CIMISS server
data = cimiss_obs_in_rect_by_time_range(
time_range, limit, data_code='SURF_CHN_MUL_MON',
elements=elements)
if data is None:
continue
# save observation data to file
out_files.append(outfile)
data.to_pickle(out_files[-1])
return out_files
def get_cmpas_hist_files(time_range, outdir='.'):
"""
Download CMAPS QPE gridded data files.
Arguments:
time_range {string} -- time range for retrieve,
"[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
outdir {string} -- output directory.
:Exampels:
>>> time_range = "[20180101000000,20180331230000]"
>>> get_cmpas_hist_files(time_range, outdir='G:/CMAPS')
"""
# check output directory
if not os.path.exists(outdir):
os.makedirs(outdir)
files = cimiss_obs_file_by_time_range(
time_range, data_code="SURF_CMPA_NRT_NC")
filenames = files['DS']
for file in filenames:
outfile = os.path.join(outdir, file['FILE_NAME'])
if not os.path.isfile(outfile):
urllib.request.urlretrieve(file['FILE_URL'], outfile)
# -*- coding: utf-8 -*-
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Retrieve historical data from CIMISS service.
"""
import os
import calendar
import urllib.request
import numpy as np
from nmc_met_io.retrieve_cimiss_server import cimiss_obs_by_time_range
from nmc_met_io.retrieve_cimiss_server import cimiss_obs_in_rect_by_time_range
from nmc_met_io.retrieve_cimiss_server import cimiss_obs_file_by_time_range
def get_day_hist_obs(years=np.arange(2000, 2011, 1),
month_range=(1, 12),
elements=None,
sta_levels=None,
outfname='day_rain_obs',
outdir='.'):
"""
Download historical daily observations and write to data files,
each month a file.
:param years: years for historical data
:param month_range: month range each year, like (1, 12)
:param elements: elements for retrieve, 'ele1, ele2, ...'
:param sta_levels: station levels
:param outfname: output file name + '_year' + '_month'
:param outdir: output file directory
:return: output file names.
:Example:
>>> get_day_hist_obs(years=np.arange(2000, 2016, 1), outdir="D:/")
"""
# check elements
if elements is None:
elements = "Station_Id_C,Station_Name,Datetime,Lat,Lon,PRE_Time_0808"
# check output directory
if not os.path.exists(outdir):
os.makedirs(outdir)
# define months
months = np.arange(1, 13, 1)
# Because of the CIMISS data mount limit,
# so loop every year to download the data.
out_files = []
for iy in years:
if calendar.isleap(iy):
last_day = ['31', '29', '31', '30', '31', '30',
'31', '31', '30', '31', '30', '31']
else:
last_day = ['31', '28', '31', '30', '31', '30',
'31', '31', '30', '31', '30', '31']
for i, im in enumerate(months):
# check month range
if not (month_range[0] <= im <= month_range[1]):
continue
month = '%02d' % im
start_time = str(iy) + month + '01' + '000000'
end_time = str(iy) + month + last_day[i] + '000000'
time_range = "[" + start_time + "," + end_time + "]"
# retrieve observations from CIMISS server
data = cimiss_obs_by_time_range(
time_range, sta_levels=sta_levels,
data_code="SURF_CHN_MUL_DAY", elements=elements)
if data is None:
continue
# save observation data to file
out_files.append(os.path.join(
outdir, outfname + "_" + str(iy) + "_" + month + ".pkl"))
data.to_pickle(out_files[-1])
return out_files
def get_mon_hist_obs(years=np.arange(2000, 2011, 1),
limit=(3, 73, 54, 136),
elements=None,
outfname='mon_surface_obs',
outdir='.'):
"""
Download historical monthly observations and write to data files,
each year a file.
:param years: years for historical data
:param limit: spatial limit [min_lat, min_lon, max_lat, max_lon]
:param elements: elements for retrieve, 'ele1, ele2, ...'
:param outfname: output file name + 'year'
:param outdir: output file directory
:return: Output filenames
"""
# check elements
if elements is None:
elements = ("Station_Id_C,Station_Name,Year,"
"Mon,Lat,Lon,Alti,PRE_Time_0808")
# check output directory
if not os.path.exists(outdir):
os.makedirs(outdir)
# Loop every year to download the data.
out_files = []
for iy in years:
# check out file
outfile = os.path.join(outdir, outfname + "_" + str(iy) + ".pkl")
if os.path.isfile(outfile):
continue
# set time range
start_time = str(iy) + '0101' + '000000'
end_time = str(iy) + '1201' + '000000'
time_range = "[" + start_time + "," + end_time + "]"
# retrieve observations from CIMISS server
data = cimiss_obs_in_rect_by_time_range(
time_range, limit, data_code='SURF_CHN_MUL_MON',
elements=elements)
if data is None:
continue
# save observation data to file
out_files.append(outfile)
data.to_pickle(out_files[-1])
return out_files
def get_cmpas_hist_files(time_range, outdir='.'):
"""
Download CMAPS QPE gridded data files.
Arguments:
time_range {string} -- time range for retrieve,
"[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
outdir {string} -- output directory.
:Exampels:
>>> time_range = "[20180101000000,20180331230000]"
>>> get_cmpas_hist_files(time_range, outdir='G:/CMAPS')
"""
# check output directory
if not os.path.exists(outdir):
os.makedirs(outdir)
files = cimiss_obs_file_by_time_range(
time_range, data_code="SURF_CMPA_NRT_NC")
filenames = files['DS']
for file in filenames:
outfile = os.path.join(outdir, file['FILE_NAME'])
if not os.path.isfile(outfile):
urllib.request.urlretrieve(file['FILE_URL'], outfile)

868
nmc_met_io/retrieve_cimiss_server.py
View File

@ -1,434 +1,434 @@
# _*_ coding: utf-8 _*_
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Retrieve the CIMISS data using REST API with pure python code.
refer to:
http://10.20.76.55/cimissapiweb/MethodData_list.action
https://github.com/babybearming/CIMISSDataGet/blob/master/cimissRead_v0.1.py
"""
import json
from datetime import datetime, timedelta
import urllib3
import numpy as np
import pandas as pd
import xarray as xr
from nmc_met_io.config import _get_config_from_rcfile
def get_http_result(interface_id, params, data_format='json'):
"""
Get the http result from CIMISS REST api service.
:param interface_id: MUSIC interface id.
:param params: dictionary for MUSIC parameters.
:param data_format: MUSIC server data format.
:return:
"""
# set MUSIC server dns and user information
config = _get_config_from_rcfile()
dns = config['CIMISS']['DNS']
user_id = config['CIMISS']['USER_ID']
pwd = config['CIMISS']['PASSWORD']
# construct url
url = 'http://' + dns + '/cimiss-web/api?userId=' + user_id + \
'&pwd=' + pwd + '&interfaceId=' + interface_id
# params
for key in params:
url += '&' + key + '=' + params[key]
# data format
url += '&dataFormat=' + data_format
# request http contents
http = urllib3.PoolManager()
req = http.request('GET', url)
if req.status != 200:
print('Can not access the url: ' + url)
return None
return req.data
def cimiss_obs_by_time_range(time_range, sta_levels=None,
data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
"""
Retrieve station records from CIMISS by time and station ID.
:param time_range: time range for retrieve,
"[YYYYMMDDHHMISS, YYYYMMDDHHMISS]",
like"[201509010000,20150903060000]"
:param sta_levels: station levels, like "011,012,013" for standard,
base and general stations.
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on.
:param elements: elements for retrieve, 'ele1,ele2,...'
:return: observation records, pandas DataFrame type
:Example:
>>> time_range = "[20180219000000,20180221000000]"
>>> sta_levels = "011,012,013"
>>> data_code = "SURF_CHN_MUL_DAY"
>>> elements = "Station_Id_C,Station_Name,Datetime,Lat,Lon,PRE_Time_0808"
>>> data = cimiss_obs_by_time_range(time_range, sta_levels=sta_levels,
data_code=data_code, elements=elements)
>>> print "retrieve successfully" if data is not None else "failed"
retrieve successfully
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'timeRange': time_range}
if sta_levels is not None:
params['staLevels'] = sta_levels
# interface id
interface_id = "getSurfEleByTimeRange"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_by_time_and_id(times, data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,TEM",
sta_ids="54511"):
"""
Retrieve station records from CIMISS by time and station ID
:param times: time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on.
:param elements: elements for retrieve, 'ele1,ele2,...'
:param sta_ids: station ids, 'xxxxx,xxxxx,...'
:return: observation records, pandas DataFrame type
:Example:
>>> data = cimiss_obs_by_time_and_id('20170318000000')
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'times': times,
'staIds': sta_ids,
'orderby': "Datetime:ASC"}
# interface id
interface_id = "getSurfEleByTimeAndStaID"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_in_rect_by_time(times, limit, data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
"""
Retrieve station records from CIMISS in region by time.
:param times: times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
:param limit: [min_lat, min_lon, max_lat, max_lon]
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on
:param elements: elements for retrieve, 'ele1,ele2,...'
:return: observation records, pandas DataFrame type
:Example:
>>> data = cimiss_obs_in_rect_by_time('20170320000000', [35, 110, 45, 120])
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'times': times,
'minLat': '{:.10f}'.format(limit[0]),
'minLon': '{:.10f}'.format(limit[1]),
'maxLat': '{:.10f}'.format(limit[2]),
'maxLon': '{:.10f}'.format(limit[3]),
'orderby': "Datetime:ASC"}
# interface id
interface_id = "getSurfEleInRectByTime"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_in_rect_by_time_range(
time_range, limit,
data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
"""
Retrieve observation records from CIMISS by rect and time range.
:param time_range: time range for retrieve,
"[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
:param limit: (min_lat, min_lon, max_lat, max_lon)
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on.
:param elements: elements for retrieve, 'ele1,ele2,...'
:return: observation records, pandas DataFrame type
:Example:
>>> elements = ("Station_Id_C,Station_Id_d,Station_Name,"
"Station_levl,Datetime,Lat,Lon,PRE_Time_0808")
>>> time_range = "[20160801000000,20160801000000]"
>>> data_code = "SURF_CHN_MUL_DAY"
>>> data = cimiss_obs_in_rect_by_time_range(
time_range,[35,110,45,120], data_code=data_code,
elements=elements)
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'timeRange': time_range,
'minLat': '{:.10f}'.format(limit[0]),
'minLon': '{:.10f}'.format(limit[1]),
'maxLat': '{:.10f}'.format(limit[2]),
'maxLon': '{:.10f}'.format(limit[3]),
'orderby': "Datetime:ASC"}
# interface id
interface_id = "getSurfEleInRectByTimeRange"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_grid_by_time(
time_str, data_code="SURF_CMPA_RT_NC", fcst_ele="PRE"):
"""
Retrieve surface analysis grid products,
like CMPAS-V2.1融合降水分析实时数据产品（NC）.
For SURF_CMPA_RT_NC, this function will retrieve
the 0.01 resolution data and take long time.
:param time_str: analysis time string, like "2017100800"
:param data_code: data code
:param fcst_ele: elements
:return: data, xarray type
:Example:
>>> time_str = "2017110612"
>>> data_code = "SURF_CMPA_RT_NC"
>>> data = cimiss_obs_grid_by_time(time_str, data_code=data_code,
fcst_ele="PRE")
"""
# set retrieve parameters
params = {'dataCode': data_code,
'time': time_str + "0000",
'fcstEle': fcst_ele}
# set interface id
interface_id = "getSurfEleGridByTime"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# get time information
time = datetime.strptime(time_str, '%Y%m%d%H')
# extract coordinates and data
start_lat = float(contents['startLat'])
start_lon = float(contents['startLon'])
nlon = int(contents['lonCount'])
nlat = int(contents['latCount'])
dlon = float(contents['lonStep'])
dlat = float(contents['latStep'])
lon = start_lon + np.arange(nlon) * dlon
lat = start_lat + np.arange(nlat) * dlat
name = contents['fieldNames']
units = contents['fieldUnits']
# construct xarray
data = np.array(contents['DS'])
data = data[np.newaxis, ...]
data = xr.DataArray(data, coords=[time, lat, lon],
dims=['time', 'lat', 'lon'], name=name)
# add attributes
data.attrs['units'] = units
data.attrs['organization'] = 'Created by NMC.'
# return data
return data
def cimiss_obs_file_by_time_range(time_range,
data_code="SURF_CMPA_RT_NC"):
"""
Retrieve CIMISS data file information.
:param time_range: time range for retrieve,
"[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
:param data_code: data code
:return: dictionary
:Examples:
>>> time_range = "[20180401000000,20180402000000]"
>>> files = cimiss_obs_file_by_time_range(time_range)
>>> filenames = files['DS']
>>> print(files['DS'][0]['FILE_URL'])
"""
# set retrieve parameters
params = {'dataCode': data_code,
'timeRange': time_range}
# set interface id
interface_id = "getSurfFileByTimeRange"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# return
return contents
def cimiss_model_by_time(init_time_str, limit=None,
data_code='NAFP_FOR_FTM_HIGH_EC_GLB',
fcst_level=0, valid_time=0, fcst_ele="TEF2"):
"""
Retrieve grid data from CIMISS service.
:param init_time_str: model run time, like "2016081712"
:param limit: [min_lat, min_lon, max_lat, max_lon]
:param data_code: MUSIC data code, default is "NAFP_FOR_FTM_HIGH_EC_GLB"
:param fcst_level: vertical level, default is 0.
:param valid_time: forecast element, default is 2m temperature "TEF2"
:param fcst_ele: forecast hour, default is 0
:return:
"""
# set retrieve parameters
if limit is None:
params = {'dataCode': data_code,
'time': init_time_str + '0000',
'fcstLevel': '{:d}'.format(fcst_level),
'validTime': '{:d}'.format(valid_time),
'fcstEle': fcst_ele}
interface_id = 'getNafpEleGridByTimeAndLevelAndValidtime'
else:
params = {'dataCode': data_code,
'time': init_time_str + '0000',
'minLat': '{:.10f}'.format(limit[0]),
"minLon": '{:.10f}'.format(limit[1]),
"maxLat": '{:.10f}'.format(limit[2]),
"maxLon": '{:.10f}'.format(limit[3]),
'fcstLevel': '{:d}'.format(fcst_level),
'validTime': '{:d}'.format(valid_time),
'fcstEle': fcst_ele}
interface_id = 'getNafpEleGridInRectByTimeAndLevelAndValidtime'
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# get time information
init_time = datetime.strptime(init_time_str, '%Y%m%d%H')
fhour = valid_time
time = init_time + timedelta(hours=fhour)
# extract coordinates and data
start_lat = float(contents['startLat'])
start_lon = float(contents['startLon'])
nlon = int(contents['lonCount'])
nlat = int(contents['latCount'])
dlon = float(contents['lonStep'])
dlat = float(contents['latStep'])
lon = start_lon + np.arange(nlon)*dlon
lat = start_lat + np.arange(nlat)*dlat
name = contents['fieldNames']
units = contents['fieldUnits']
# construct xarray
data = np.array(contents['DS'])
if fcst_level == 0:
data = data[np.newaxis, ...]
data = xr.DataArray(data, coords=[time, lat, lon],
dims=['time', 'lat', 'lon'], name=name)
else:
data = data[np.newaxis, np.newaxis, ...]
data = xr.DataArray(data, coords=[time, fcst_level, lat, lon],
dims=['time', 'level', 'lat', 'lon'], name=name)
# add time coordinates
data.coords['init_time'] = ('time', init_time)
data.coords['fhour'] = ('time', fhour)
# add attributes
data.attrs['units'] = units
data.attrs['organization'] = 'Created by NMC.'
# return data
return data
# _*_ coding: utf-8 _*_
# Copyright (c) 2019 NMC Developers.
# Distributed under the terms of the GPL V3 License.
"""
Retrieve the CIMISS data using REST API with pure python code.
refer to:
http://10.20.76.55/cimissapiweb/MethodData_list.action
https://github.com/babybearming/CIMISSDataGet/blob/master/cimissRead_v0.1.py
"""
import json
from datetime import datetime, timedelta
import urllib3
import numpy as np
import pandas as pd
import xarray as xr
from nmc_met_io.config import _get_config_from_rcfile
def get_http_result(interface_id, params, data_format='json'):
"""
Get the http result from CIMISS REST api service.
:param interface_id: MUSIC interface id.
:param params: dictionary for MUSIC parameters.
:param data_format: MUSIC server data format.
:return:
"""
# set MUSIC server dns and user information
config = _get_config_from_rcfile()
dns = config['CIMISS']['DNS']
user_id = config['CIMISS']['USER_ID']
pwd = config['CIMISS']['PASSWORD']
# construct url
url = 'http://' + dns + '/cimiss-web/api?userId=' + user_id + \
'&pwd=' + pwd + '&interfaceId=' + interface_id
# params
for key in params:
url += '&' + key + '=' + params[key]
# data format
url += '&dataFormat=' + data_format
# request http contents
http = urllib3.PoolManager()
req = http.request('GET', url)
if req.status != 200:
print('Can not access the url: ' + url)
return None
return req.data
def cimiss_obs_by_time_range(time_range, sta_levels=None,
data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
"""
Retrieve station records from CIMISS by time and station ID.
:param time_range: time range for retrieve,
"[YYYYMMDDHHMISS, YYYYMMDDHHMISS]",
like"[201509010000,20150903060000]"
:param sta_levels: station levels, like "011,012,013" for standard,
base and general stations.
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on.
:param elements: elements for retrieve, 'ele1,ele2,...'
:return: observation records, pandas DataFrame type
:Example:
>>> time_range = "[20180219000000,20180221000000]"
>>> sta_levels = "011,012,013"
>>> data_code = "SURF_CHN_MUL_DAY"
>>> elements = "Station_Id_C,Station_Name,Datetime,Lat,Lon,PRE_Time_0808"
>>> data = cimiss_obs_by_time_range(time_range, sta_levels=sta_levels,
data_code=data_code, elements=elements)
>>> print "retrieve successfully" if data is not None else "failed"
retrieve successfully
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'timeRange': time_range}
if sta_levels is not None:
params['staLevels'] = sta_levels
# interface id
interface_id = "getSurfEleByTimeRange"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_by_time_and_id(times, data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,TEM",
sta_ids="54511"):
"""
Retrieve station records from CIMISS by time and station ID
:param times: time for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on.
:param elements: elements for retrieve, 'ele1,ele2,...'
:param sta_ids: station ids, 'xxxxx,xxxxx,...'
:return: observation records, pandas DataFrame type
:Example:
>>> data = cimiss_obs_by_time_and_id('20170318000000')
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'times': times,
'staIds': sta_ids,
'orderby': "Datetime:ASC"}
# interface id
interface_id = "getSurfEleByTimeAndStaID"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_in_rect_by_time(times, limit, data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
"""
Retrieve station records from CIMISS in region by time.
:param times: times for retrieve, 'YYYYMMDDHHMISS,YYYYMMDDHHMISS,...'
:param limit: [min_lat, min_lon, max_lat, max_lon]
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on
:param elements: elements for retrieve, 'ele1,ele2,...'
:return: observation records, pandas DataFrame type
:Example:
>>> data = cimiss_obs_in_rect_by_time('20170320000000', [35, 110, 45, 120])
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'times': times,
'minLat': '{:.10f}'.format(limit[0]),
'minLon': '{:.10f}'.format(limit[1]),
'maxLat': '{:.10f}'.format(limit[2]),
'maxLon': '{:.10f}'.format(limit[3]),
'orderby': "Datetime:ASC"}
# interface id
interface_id = "getSurfEleInRectByTime"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_in_rect_by_time_range(
time_range, limit,
data_code="SURF_CHN_MUL_HOR_N",
elements="Station_Id_C,Datetime,Lat,Lon,TEM"):
"""
Retrieve observation records from CIMISS by rect and time range.
:param time_range: time range for retrieve,
"[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
:param limit: (min_lat, min_lon, max_lat, max_lon)
:param data_code: dataset code, like "SURF_CHN_MUL_HOR",
"SURF_CHN_MUL_HOR_N", and so on.
:param elements: elements for retrieve, 'ele1,ele2,...'
:return: observation records, pandas DataFrame type
:Example:
>>> elements = ("Station_Id_C,Station_Id_d,Station_Name,"
"Station_levl,Datetime,Lat,Lon,PRE_Time_0808")
>>> time_range = "[20160801000000,20160801000000]"
>>> data_code = "SURF_CHN_MUL_DAY"
>>> data = cimiss_obs_in_rect_by_time_range(
time_range,[35,110,45,120], data_code=data_code,
elements=elements)
"""
# set retrieve parameters
params = {'dataCode': data_code,
'elements': elements,
'timeRange': time_range,
'minLat': '{:.10f}'.format(limit[0]),
'minLon': '{:.10f}'.format(limit[1]),
'maxLat': '{:.10f}'.format(limit[2]),
'maxLon': '{:.10f}'.format(limit[3]),
'orderby': "Datetime:ASC"}
# interface id
interface_id = "getSurfEleInRectByTimeRange"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# construct pandas DataFrame
data = pd.DataFrame(contents['DS'])
# return
return data
def cimiss_obs_grid_by_time(
time_str, data_code="SURF_CMPA_RT_NC", fcst_ele="PRE"):
"""
Retrieve surface analysis grid products,
like CMPAS-V2.1融合降水分析实时数据产品（NC）.
For SURF_CMPA_RT_NC, this function will retrieve
the 0.01 resolution data and take long time.
:param time_str: analysis time string, like "2017100800"
:param data_code: data code
:param fcst_ele: elements
:return: data, xarray type
:Example:
>>> time_str = "2017110612"
>>> data_code = "SURF_CMPA_RT_NC"
>>> data = cimiss_obs_grid_by_time(time_str, data_code=data_code,
fcst_ele="PRE")
"""
# set retrieve parameters
params = {'dataCode': data_code,
'time': time_str + "0000",
'fcstEle': fcst_ele}
# set interface id
interface_id = "getSurfEleGridByTime"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# get time information
time = datetime.strptime(time_str, '%Y%m%d%H')
# extract coordinates and data
start_lat = float(contents['startLat'])
start_lon = float(contents['startLon'])
nlon = int(contents['lonCount'])
nlat = int(contents['latCount'])
dlon = float(contents['lonStep'])
dlat = float(contents['latStep'])
lon = start_lon + np.arange(nlon) * dlon
lat = start_lat + np.arange(nlat) * dlat
name = contents['fieldNames']
units = contents['fieldUnits']
# construct xarray
data = np.array(contents['DS'])
data = data[np.newaxis, ...]
data = xr.DataArray(data, coords=[time, lat, lon],
dims=['time', 'lat', 'lon'], name=name)
# add attributes
data.attrs['units'] = units
data.attrs['organization'] = 'Created by NMC.'
# return data
return data
def cimiss_obs_file_by_time_range(time_range,
data_code="SURF_CMPA_RT_NC"):
"""
Retrieve CIMISS data file information.
:param time_range: time range for retrieve,
"[YYYYMMDDHHMISS,YYYYMMDDHHMISS]"
:param data_code: data code
:return: dictionary
:Examples:
>>> time_range = "[20180401000000,20180402000000]"
>>> files = cimiss_obs_file_by_time_range(time_range)
>>> filenames = files['DS']
>>> print(files['DS'][0]['FILE_URL'])
"""
# set retrieve parameters
params = {'dataCode': data_code,
'timeRange': time_range}
# set interface id
interface_id = "getSurfFileByTimeRange"
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# return
return contents
def cimiss_model_by_time(init_time_str, limit=None,
data_code='NAFP_FOR_FTM_HIGH_EC_GLB',
fcst_level=0, valid_time=0, fcst_ele="TEF2"):
"""
Retrieve grid data from CIMISS service.
:param init_time_str: model run time, like "2016081712"
:param limit: [min_lat, min_lon, max_lat, max_lon]
:param data_code: MUSIC data code, default is "NAFP_FOR_FTM_HIGH_EC_GLB"
:param fcst_level: vertical level, default is 0.
:param valid_time: forecast element, default is 2m temperature "TEF2"
:param fcst_ele: forecast hour, default is 0
:return:
"""
# set retrieve parameters
if limit is None:
params = {'dataCode': data_code,
'time': init_time_str + '0000',
'fcstLevel': '{:d}'.format(fcst_level),
'validTime': '{:d}'.format(valid_time),
'fcstEle': fcst_ele}
interface_id = 'getNafpEleGridByTimeAndLevelAndValidtime'
else:
params = {'dataCode': data_code,
'time': init_time_str + '0000',
'minLat': '{:.10f}'.format(limit[0]),
"minLon": '{:.10f}'.format(limit[1]),
"maxLat": '{:.10f}'.format(limit[2]),
"maxLon": '{:.10f}'.format(limit[3]),
'fcstLevel': '{:d}'.format(fcst_level),
'validTime': '{:d}'.format(valid_time),
'fcstEle': fcst_ele}
interface_id = 'getNafpEleGridInRectByTimeAndLevelAndValidtime'
# retrieve data contents
contents = get_http_result(interface_id, params)
if contents is None:
return None
contents = json.loads(contents.decode('utf-8'))
if contents['returnCode'] != '0':
return None
# get time information
init_time = datetime.strptime(init_time_str, '%Y%m%d%H')
fhour = valid_time
time = init_time + timedelta(hours=fhour)
# extract coordinates and data
start_lat = float(contents['startLat'])
start_lon = float(contents['startLon'])
nlon = int(contents['lonCount'])
nlat = int(contents['latCount'])
dlon = float(contents['lonStep'])
dlat = float(contents['latStep'])
lon = start_lon + np.arange(nlon)*dlon
lat = start_lat + np.arange(nlat)*dlat
name = contents['fieldNames']
units = contents['fieldUnits']
# construct xarray
data = np.array(contents['DS'])
if fcst_level == 0:
data = data[np.newaxis, ...]
data = xr.DataArray(data, coords=[time, lat, lon],
dims=['time', 'lat', 'lon'], name=name)
else:
data = data[np.newaxis, np.newaxis, ...]
data = xr.DataArray(data, coords=[time, fcst_level, lat, lon],
dims=['time', 'level', 'lat', 'lon'], name=name)
# add time coordinates
data.coords['init_time'] = ('time', init_time)
data.coords['fhour'] = ('time', fhour)
# add attributes
data.attrs['units'] = units
data.attrs['organization'] = 'Created by NMC.'
# return data
return data

1571
nmc_met_io/retrieve_micaps_server.py
View File

File diff suppressed because it is too large Load Diff

126
setup.py
View File

@ -1,63 +1,63 @@
# _*_ coding: utf-8 _*_
from os import path
from setuptools import find_packages, setup
from codecs import open
name = 'nmc_met_io'
author = __import__(name).__author__
version = __import__(name).__version__
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
setup(
name=name,
version=version,
description=("Collection of tools for I/O or"
"accessing meteorological data."),
long_description=long_description,
# author
author=author,
author_email='kan.dai@foxmail.com',
# LICENSE
license='GPL3',
classifiers=[
'Development Status :: 3 - Alpha',
'Intended Audience :: Developers',
'Programming Language :: Python :: 3',
],
packages=find_packages(exclude=['docs', 'tests', 'build', 'dist']),
include_package_data=True,
exclude_package_data={'': ['.gitignore']},
install_requires=['numpy>=1.12.1',
'scipy>=0.19.0',
'xarray>=0.9.6',
'pandas>=0.20.0',
'pyproj>=1.9.5.1',
'protobuf>=3.5.0',
'urllib3>=1.20',
'python-dateutil']
)
# development mode (DOS command):
# python setup.py develop
# python setup.py develop --uninstall
# build mode：
# python setup.py build --build-base=D:/test/python/build
# distribution mode:
# python setup.py sdist # create source tar.gz file in /dist
# python setup.py bdist_wheel # create wheel binary in /dist
# _*_ coding: utf-8 _*_
from os import path
from setuptools import find_packages, setup
from codecs import open
name = 'nmc_met_io'
author = __import__(name).__author__
version = __import__(name).__version__
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
setup(
name=name,
version=version,
description=("Collection of tools for I/O or"
"accessing meteorological data."),
long_description=long_description,
# author
author=author,
author_email='kan.dai@foxmail.com',
# LICENSE
license='GPL3',
classifiers=[
'Development Status :: 3 - Alpha',
'Intended Audience :: Developers',
'Programming Language :: Python :: 3',
],
packages=find_packages(exclude=['docs', 'tests', 'build', 'dist']),
include_package_data=True,
exclude_package_data={'': ['.gitignore']},
install_requires=['numpy>=1.12.1',
'scipy>=0.19.0',
'xarray>=0.9.6',
'pandas>=0.20.0',
'pyproj>=1.9.5.1',
'protobuf>=3.5.0',
'urllib3>=1.20',
'python-dateutil']
)
# development mode (DOS command):
# python setup.py develop
# python setup.py develop --uninstall
# build mode：
# python setup.py build --build-base=D:/test/python/build
# distribution mode:
# python setup.py sdist # create source tar.gz file in /dist
# python setup.py bdist_wheel # create wheel binary in /dist