Skip to content

Commit f409b26

Browse files
author
继盛
committed
upgrade to 0.8.0
1 parent fe7bc61 commit f409b26

37 files changed

+1411
-360
lines changed

‎docs/source/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,5 @@ PyODPS 的相关依赖会自动安装。
5555
interactive
5656
options
5757
faq
58+
mars-int
5859
api

‎docs/source/locale/en/LC_MESSAGES/api-df.po

Lines changed: 55 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ msgid ""
88
msgstr ""
99
"Project-Id-Version: PyODPS 0.7.16\n"
1010
"Report-Msgid-Bugs-To: \n"
11-
"POT-Creation-Date: 2018-11-08 12:41+0800\n"
11+
"POT-Creation-Date: 2018-12-05 22:57+0800\n"
1212
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
1313
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
1414
"Language-Team: LANGUAGE <[email protected]>\n"
@@ -890,20 +890,33 @@ msgstr ""
890890
msgid "instance priority, 9 as default"
891891
msgstr ""
892892

893-
#: odps.df.CollectionExpr.execute:5 odps.df.RandomScalar.execute:5
894-
#: odps.df.Scalar.execute:5 odps.df.SequenceExpr.execute:5
893+
#: odps.df.CollectionExpr.execute:5 odps.df.CollectionExpr.persist:19
894+
#: odps.df.RandomScalar.execute:5 odps.df.RandomScalar.persist:19
895+
#: odps.df.Scalar.execute:5 odps.df.Scalar.persist:19
896+
#: odps.df.SequenceExpr.execute:5 odps.df.SequenceExpr.persist:19
895897
#: odps.df.expr.groupby.GroupBy.execute:5
898+
#: odps.df.expr.groupby.GroupBy.persist:19
896899
#: odps.df.expr.groupby.Int64SequenceGroupBy.execute:5
897-
#: odps.df.expr.groupby.SequenceGroupBy.execute:5 of
898-
msgid "execution result"
900+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:19
901+
#: odps.df.expr.groupby.SequenceGroupBy.execute:5
902+
#: odps.df.expr.groupby.SequenceGroupBy.persist:19 of
903+
msgid "cluster to run this instance"
899904
msgstr ""
900905

901-
#: odps.df.CollectionExpr.execute:6 odps.df.CollectionExpr.head:5
902-
#: odps.df.CollectionExpr.tail:5 odps.df.RandomScalar.execute:6
906+
#: odps.df.CollectionExpr.execute:6 odps.df.RandomScalar.execute:6
903907
#: odps.df.Scalar.execute:6 odps.df.SequenceExpr.execute:6
904908
#: odps.df.expr.groupby.GroupBy.execute:6
905909
#: odps.df.expr.groupby.Int64SequenceGroupBy.execute:6
906910
#: odps.df.expr.groupby.SequenceGroupBy.execute:6 of
911+
msgid "execution result"
912+
msgstr ""
913+
914+
#: odps.df.CollectionExpr.execute:7 odps.df.CollectionExpr.head:5
915+
#: odps.df.CollectionExpr.tail:5 odps.df.RandomScalar.execute:7
916+
#: odps.df.Scalar.execute:7 odps.df.SequenceExpr.execute:7
917+
#: odps.df.expr.groupby.GroupBy.execute:7
918+
#: odps.df.expr.groupby.Int64SequenceGroupBy.execute:7
919+
#: odps.df.expr.groupby.SequenceGroupBy.execute:7 of
907920
msgid ":class:`odps.df.backends.frame.ResultFrame`"
908921
msgstr ""
909922

@@ -1364,59 +1377,59 @@ msgstr ""
13641377
msgid "project name, if not provided, will be the default project"
13651378
msgstr ""
13661379

1367-
#: odps.df.CollectionExpr.persist:19 odps.df.RandomScalar.persist:19
1368-
#: odps.df.Scalar.persist:19 odps.df.SequenceExpr.persist:19
1369-
#: odps.df.expr.groupby.GroupBy.persist:19
1370-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:19
1371-
#: odps.df.expr.groupby.SequenceGroupBy.persist:19 of
1380+
#: odps.df.CollectionExpr.persist:20 odps.df.RandomScalar.persist:20
1381+
#: odps.df.Scalar.persist:20 odps.df.SequenceExpr.persist:20
1382+
#: odps.df.expr.groupby.GroupBy.persist:20
1383+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:20
1384+
#: odps.df.expr.groupby.SequenceGroupBy.persist:20 of
13721385
msgid "overwrite the table, True as default"
13731386
msgstr ""
13741387

1375-
#: odps.df.CollectionExpr.persist:21 odps.df.RandomScalar.persist:21
1376-
#: odps.df.Scalar.persist:21 odps.df.SequenceExpr.persist:21
1377-
#: odps.df.expr.groupby.GroupBy.persist:21
1378-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:21
1379-
#: odps.df.expr.groupby.SequenceGroupBy.persist:21 of
1388+
#: odps.df.CollectionExpr.persist:22 odps.df.RandomScalar.persist:22
1389+
#: odps.df.Scalar.persist:22 odps.df.SequenceExpr.persist:22
1390+
#: odps.df.expr.groupby.GroupBy.persist:22
1391+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:22
1392+
#: odps.df.expr.groupby.SequenceGroupBy.persist:22 of
13801393
msgid "drop table if exists, False as default"
13811394
msgstr ""
13821395

1383-
#: odps.df.CollectionExpr.persist:23 odps.df.RandomScalar.persist:23
1384-
#: odps.df.Scalar.persist:23 odps.df.SequenceExpr.persist:23
1385-
#: odps.df.expr.groupby.GroupBy.persist:23
1386-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:23
1387-
#: odps.df.expr.groupby.SequenceGroupBy.persist:23 of
1396+
#: odps.df.CollectionExpr.persist:24 odps.df.RandomScalar.persist:24
1397+
#: odps.df.Scalar.persist:24 odps.df.SequenceExpr.persist:24
1398+
#: odps.df.expr.groupby.GroupBy.persist:24
1399+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:24
1400+
#: odps.df.expr.groupby.SequenceGroupBy.persist:24 of
13881401
msgid "create table first if not exits, True as default"
13891402
msgstr ""
13901403

1391-
#: odps.df.CollectionExpr.persist:25 odps.df.RandomScalar.persist:25
1392-
#: odps.df.Scalar.persist:25 odps.df.SequenceExpr.persist:25
1393-
#: odps.df.expr.groupby.GroupBy.persist:25
1394-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:25
1395-
#: odps.df.expr.groupby.SequenceGroupBy.persist:25 of
1404+
#: odps.df.CollectionExpr.persist:26 odps.df.RandomScalar.persist:26
1405+
#: odps.df.Scalar.persist:26 odps.df.SequenceExpr.persist:26
1406+
#: odps.df.expr.groupby.GroupBy.persist:26
1407+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:26
1408+
#: odps.df.expr.groupby.SequenceGroupBy.persist:26 of
13961409
msgid "drop partition if exists, False as default"
13971410
msgstr ""
13981411

1399-
#: odps.df.CollectionExpr.persist:27 odps.df.RandomScalar.persist:27
1400-
#: odps.df.Scalar.persist:27 odps.df.SequenceExpr.persist:27
1401-
#: odps.df.expr.groupby.GroupBy.persist:27
1402-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:27
1403-
#: odps.df.expr.groupby.SequenceGroupBy.persist:27 of
1412+
#: odps.df.CollectionExpr.persist:28 odps.df.RandomScalar.persist:28
1413+
#: odps.df.Scalar.persist:28 odps.df.SequenceExpr.persist:28
1414+
#: odps.df.expr.groupby.GroupBy.persist:28
1415+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:28
1416+
#: odps.df.expr.groupby.SequenceGroupBy.persist:28 of
14041417
msgid "create partition if not exists, None as default"
14051418
msgstr ""
14061419

1407-
#: odps.df.CollectionExpr.persist:29 odps.df.RandomScalar.persist:29
1408-
#: odps.df.Scalar.persist:29 odps.df.SequenceExpr.persist:29
1409-
#: odps.df.expr.groupby.GroupBy.persist:29
1410-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:29
1411-
#: odps.df.expr.groupby.SequenceGroupBy.persist:29 of
1420+
#: odps.df.CollectionExpr.persist:30 odps.df.RandomScalar.persist:30
1421+
#: odps.df.Scalar.persist:30 odps.df.SequenceExpr.persist:30
1422+
#: odps.df.expr.groupby.GroupBy.persist:30
1423+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:30
1424+
#: odps.df.expr.groupby.SequenceGroupBy.persist:30 of
14121425
msgid "cast all columns' types as the existed table, False as default"
14131426
msgstr ""
14141427

1415-
#: odps.df.CollectionExpr.persist:31 odps.df.RandomScalar.persist:31
1416-
#: odps.df.Scalar.persist:31 odps.df.SequenceExpr.persist:31
1417-
#: odps.df.expr.groupby.GroupBy.persist:31
1418-
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:31
1419-
#: odps.df.expr.groupby.SequenceGroupBy.persist:31 of
1428+
#: odps.df.CollectionExpr.persist:32 odps.df.RandomScalar.persist:32
1429+
#: odps.df.Scalar.persist:32 odps.df.SequenceExpr.persist:32
1430+
#: odps.df.expr.groupby.GroupBy.persist:32
1431+
#: odps.df.expr.groupby.Int64SequenceGroupBy.persist:32
1432+
#: odps.df.expr.groupby.SequenceGroupBy.persist:32 of
14201433
msgid ":class:`odps.df.DataFrame`"
14211434
msgstr ""
14221435

‎docs/source/locale/en/LC_MESSAGES/options.po

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,16 @@
66
#, fuzzy
77
msgid ""
88
msgstr ""
9-
"Project-Id-Version: PyODPS 0.7.16\n"
9+
"Project-Id-Version: PyODPS 0.8.0\n"
1010
"Report-Msgid-Bugs-To: \n"
11-
"POT-Creation-Date: 2018-06-21 09:15+0800\n"
11+
"POT-Creation-Date: 2018-12-05 11:38+0800\n"
1212
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
1313
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
1414
"Language-Team: LANGUAGE <[email protected]>\n"
1515
"MIME-Version: 1.0\n"
1616
"Content-Type: text/plain; charset=utf-8\n"
1717
"Content-Transfer-Encoding: 8bit\n"
18-
"Generated-By: Babel 2.5.3\n"
18+
"Generated-By: Babel 2.6.0\n"
1919

2020
#: ../../source/options.rst:5
2121
msgid "配置选项"
@@ -79,12 +79,12 @@ msgid "ODPS Endpoint"
7979
msgstr "MaxCompute Endpoint"
8080

8181
#: ../../source/options.rst:27 ../../source/options.rst:29
82-
#: ../../source/options.rst:31 ../../source/options.rst:38
83-
#: ../../source/options.rst:42 ../../source/options.rst:46
84-
#: ../../source/options.rst:60 ../../source/options.rst:62
85-
#: ../../source/options.rst:68 ../../source/options.rst:79
86-
#: ../../source/options.rst:81 ../../source/options.rst:98
87-
#: ../../source/options.rst:110
82+
#: ../../source/options.rst:31 ../../source/options.rst:35
83+
#: ../../source/options.rst:38 ../../source/options.rst:42
84+
#: ../../source/options.rst:46 ../../source/options.rst:60
85+
#: ../../source/options.rst:62 ../../source/options.rst:68
86+
#: ../../source/options.rst:79 ../../source/options.rst:81
87+
#: ../../source/options.rst:98 ../../source/options.rst:110
8888
msgid "None"
8989
msgstr ""
9090

@@ -121,14 +121,11 @@ msgid "local_timezone"
121121
msgstr ""
122122

123123
#: ../../source/options.rst:35
124-
msgid "使用的时区，True 表示本地时间，False 表示 UTC， 也可用 pytz 的时区"
125-
msgstr ""
126-
"Used time zone. True indicates local time, and False indicates UTC. The "
127-
"time zone of pytz can also be used."
128-
129-
#: ../../source/options.rst:35 ../../source/options.rst:40
130-
msgid "1"
124+
msgid "使用的时区，None 表示不处理，True 表示本地时区， False 表示 UTC，也可用 pytz 的时区"
131125
msgstr ""
126+
"Used time zone. None indicates that PyODPS takes no actions, True "
127+
"indicates local time, and False indicates UTC. The time zone of pytz "
128+
"package can also be used."
132129

133130
#: ../../source/options.rst:38
134131
msgid "lifecycle"
@@ -146,6 +143,10 @@ msgstr ""
146143
msgid "临时表生命周期"
147144
msgstr "Life cycles of temporary tables"
148145

146+
#: ../../source/options.rst:40
147+
msgid "1"
148+
msgstr ""
149+
149150
#: ../../source/options.rst:42
150151
msgid "biz_id"
151152
msgstr ""

‎docs/source/options.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ PyODPS 提供了一系列的配置选项，可通过 ``odps.options`` 获得，
3232
+------------------------+---------------------------------------------------+-------+
3333
|log_view_hours | LogView 保持时间（小时） |24 |
3434
+------------------------+---------------------------------------------------+-------+
35-
|local_timezone | 使用的时区，True 表示本地时间，False 表示 UTC， |1 |
36-
| | 也可用 pytz 的时区 | |
35+
|local_timezone | 使用的时区，None 表示不处理，True 表示本地时区， |None |
36+
| | False 表示 UTC，也可用 pytz 的时区 | |
3737
+------------------------+---------------------------------------------------+-------+
3838
|lifecycle | 所有表生命周期 |None |
3939
+------------------------+---------------------------------------------------+-------+

‎odps/_version.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
version_info = (0, 7, 20, 1)
15+
version_info = (0, 8, 0)
1616
_num_index = max(idx if isinstance(v, int) else 0
1717
for idx, v in enumerate(version_info))
1818
__version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \

‎odps/config.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,8 @@ def validate(x):
325325
options.register_option('account', None)
326326
options.register_option('end_point', None)
327327
options.register_option('default_project', None)
328-
options.register_option('local_timezone', True)
328+
options.register_option('local_timezone', None)
329+
options.register_option('allow_antique_date', False)
329330
options.register_option('user_agent_pattern', '$pyodps_version $python_version $os_version')
330331
options.register_option('log_view_host', None)
331332
options.register_option('log_view_hours', 24, validator=is_integer)

‎odps/df/__init__.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,10 @@ def inner(*args, **kwargs):
5959
locals()[k] = wrap(v)
6060
except ImportError:
6161
pass
62+
63+
try:
64+
from ..internal import df as internal_df
65+
66+
del internal_df
67+
except ImportError:
68+
pass

‎odps/df/backends/engine.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def get_default_engine(*exprs):
7070
odps = expr_odps
7171

7272
if odps is None and options.account is not None and \
73-
options.end_point is not None and options.default_project is not None:
73+
options.end_point is not None and options.default_project is not None:
7474
odps = ODPS._from_account(options.account, options.default_project,
7575
endpoint=options.end_point,
7676
tunnel_endpoint=options.tunnel.endpoint)

‎odps/df/backends/odpssql/engine.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -129,10 +129,11 @@ def _reload_ui(self, group, instance, ui):
129129
return fetch_instance_group(group).instances.get(instance.id)
130130

131131
def _run(self, sql, ui, progress_proportion=1, hints=None, priority=None,
132-
group=None, libraries=None):
132+
running_cluster=None, group=None, libraries=None):
133133
libraries = self._ctx.prepare_resources(self._get_libraries(libraries))
134134
self._ctx.create_udfs(libraries=libraries)
135-
instance = self._odps.run_sql(sql, hints=hints, priority=priority, name='PyODPSDataFrameTask')
135+
instance = self._odps.run_sql(sql, hints=hints, priority=priority, name='PyODPSDataFrameTask',
136+
running_cluster=running_cluster)
136137

137138
self._instances.append(instance.id)
138139
log('Instance ID: ' + instance.id)
@@ -287,7 +288,7 @@ def _join_sql(cls, sql):
287288

288289
def _do_execute(self, expr_dag, expr, ui=None, progress_proportion=1,
289290
lifecycle=None, head=None, tail=None,
290-
hints=None, priority=None, **kw):
291+
hints=None, priority=None, running_cluster=None, **kw):
291292
lifecycle = lifecycle or options.temp_lifecycle
292293
group = kw.get('group')
293294
libraries = kw.pop('libraries', None)
@@ -344,7 +345,7 @@ def _do_execute(self, expr_dag, expr, ui=None, progress_proportion=1,
344345
log(sql)
345346

346347
instance = self._run(sql, ui, progress_proportion=progress_proportion*0.9,
347-
hints=hints, priority=priority,
348+
hints=hints, priority=priority, running_cluster=running_cluster,
348349
group=group, libraries=libraries)
349350

350351
self._ctx.close() # clear udfs and resources generated
@@ -433,8 +434,8 @@ def _fetch(self, expr, src_expr, instance, ui, progress_proportion=1,
433434

434435
def _do_persist(self, expr_dag, expr, name, partitions=None, partition=None, project=None, ui=None,
435436
progress_proportion=1, lifecycle=None, hints=None, priority=None,
436-
overwrite=True, drop_table=False, create_table=True, drop_partition=False,
437-
create_partition=None, cast=False, **kw):
437+
running_cluster=None, overwrite=True, drop_table=False, create_table=True,
438+
drop_partition=False, create_partition=None, cast=False, **kw):
438439
group = kw.get('group')
439440
libraries = kw.pop('libraries', None)
440441

@@ -567,7 +568,8 @@ def _do_persist(self, expr_dag, expr, name, partitions=None, partition=None, pro
567568
log(sql)
568569

569570
instance = self._run(sql, ui, progress_proportion=progress_proportion,
570-
hints=hints, priority=priority, group=group, libraries=libraries)
571+
hints=hints, priority=priority, running_cluster=running_cluster,
572+
group=group, libraries=libraries)
571573
self._ctx.close() # clear udfs and resources generated
572574
t = self._odps.get_table(name, project=project)
573575
if should_cache and not is_source_collection(src_expr):

‎odps/df/expr/expressions.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ def execute(self, **kwargs):
176176
:type hints: dict
177177
:param priority: instance priority, 9 as default
178178
:type priority: int
179+
:param running_cluster: cluster to run this instance
179180
:return: execution result
180181
:rtype: :class:`odps.df.backends.frame.ResultFrame`
181182
"""
@@ -223,6 +224,7 @@ def persist(self, name, partitions=None, partition=None, lifecycle=None, project
223224
:type hints: dict
224225
:param priority: instance priority, 9 as default
225226
:type priority: int
227+
:param running_cluster: cluster to run this instance
226228
:param overwrite: overwrite the table, True as default
227229
:type overwrite: bool
228230
:param drop_table: drop table if exists, False as default

‎odps/lib/tzlocal/LICENSE.txt

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Copyright 2011-2017 Lennart Regebro
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy
4+
of this software and associated documentation files (the "Software"), to deal
5+
in the Software without restriction, including without limitation the rights
6+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7+
copies of the Software, and to permit persons to whom the Software is
8+
furnished to do so, subject to the following conditions:
9+
10+
The above copyright notice and this permission notice shall be included in
11+
all copies or substantial portions of the Software.
12+
13+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19+
SOFTWARE.

‎odps/lib/tzlocal/__init__.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import sys
2+
if sys.platform == 'win32':
3+
from .win32 import get_localzone, reload_localzone
4+
else:
5+
from .unix import get_localzone, reload_localzone

0 commit comments

Comments
 (0)