Skip to content

Commit 3bf5aed

Browse files
alimcmaster1alimcmaster1
committed
Merge remote-tracking branch 'remotes/upstream/master' into mcmali-fixture-gbq
2 parents f4ecffb + 8c3883d commit 3bf5aed

File tree

233 files changed

+7166
-4533
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

233 files changed

+7166
-4533
lines changed

‎.github/workflows/ci.yml

+72-10
Original file line numberDiff line numberDiff line change
@@ -23,53 +23,53 @@ jobs:
2323

2424
- name: Looking for unwanted patterns
2525
run: ci/code_checks.sh patterns
26-
if: true
26+
if: always()
2727

2828
- name: Setup environment and build pandas
2929
run: ci/setup_env.sh
30-
if: true
30+
if: always()
3131

3232
- name: Linting
3333
run: |
3434
source activate pandas-dev
3535
ci/code_checks.sh lint
36-
if: true
36+
if: always()
3737

3838
- name: Dependencies consistency
3939
run: |
4040
source activate pandas-dev
4141
ci/code_checks.sh dependencies
42-
if: true
42+
if: always()
4343

4444
- name: Checks on imported code
4545
run: |
4646
source activate pandas-dev
4747
ci/code_checks.sh code
48-
if: true
48+
if: always()
4949

5050
- name: Running doctests
5151
run: |
5252
source activate pandas-dev
5353
ci/code_checks.sh doctests
54-
if: true
54+
if: always()
5555

5656
- name: Docstring validation
5757
run: |
5858
source activate pandas-dev
5959
ci/code_checks.sh docstrings
60-
if: true
60+
if: always()
6161

6262
- name: Typing validation
6363
run: |
6464
source activate pandas-dev
6565
ci/code_checks.sh typing
66-
if: true
66+
if: always()
6767

6868
- name: Testing docstring validation script
6969
run: |
7070
source activate pandas-dev
7171
pytest --capture=no --strict scripts
72-
if: true
72+
if: always()
7373

7474
- name: Running benchmarks
7575
run: |
@@ -87,11 +87,73 @@ jobs:
8787
else
8888
echo "Benchmarks did not run, no changes detected"
8989
fi
90-
if: true
90+
if: always()
9191

9292
- name: Publish benchmarks artifact
9393
uses: actions/upload-artifact@master
9494
with:
9595
name: Benchmarks log
9696
path: asv_bench/benchmarks.log
9797
if: failure()
98+
99+
web_and_docs:
100+
name: Web and docs
101+
runs-on: ubuntu-latest
102+
steps:
103+
104+
- name: Setting conda path
105+
run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
106+
107+
- name: Checkout
108+
uses: actions/checkout@v1
109+
110+
- name: Setup environment and build pandas
111+
run: ci/setup_env.sh
112+
113+
- name: Build website
114+
run: |
115+
source activate pandas-dev
116+
python web/pandas_web.py web/pandas --target-path=web/build
117+
118+
- name: Build documentation
119+
run: |
120+
source activate pandas-dev
121+
doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]}
122+
123+
# This can be removed when the ipython directive fails when there are errors,
124+
# including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
125+
- name: Check ipython directive errors
126+
run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
127+
128+
- name: Merge website and docs
129+
run: |
130+
mkdir -p pandas_web/docs
131+
cp -r web/build/* pandas_web/
132+
cp -r doc/build/html/* pandas_web/docs/
133+
if: github.event_name == 'push'
134+
135+
- name: Install Rclone
136+
run: sudo apt install rclone -y
137+
if: github.event_name == 'push'
138+
139+
- name: Set up Rclone
140+
run: |
141+
RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf
142+
mkdir -p `dirname $RCLONE_CONFIG_PATH`
143+
echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH
144+
echo "type = swift" >> $RCLONE_CONFIG_PATH
145+
echo "env_auth = false" >> $RCLONE_CONFIG_PATH
146+
echo "auth_version = 3" >> $RCLONE_CONFIG_PATH
147+
echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH
148+
echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH
149+
echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH
150+
echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH
151+
echo "domain = default" >> $RCLONE_CONFIG_PATH
152+
echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH
153+
echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH
154+
echo "region = BHS" >> $RCLONE_CONFIG_PATH
155+
if: github.event_name == 'push'
156+
157+
- name: Sync web
158+
run: rclone sync pandas_web ovh_cloud_pandas_web:dev
159+
if: github.event_name == 'push'

‎.travis.yml

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
language: python
2-
python: 3.5
2+
python: 3.7
33

44
# To turn off cached cython files and compiler cache
55
# set NOCACHE-true
@@ -48,17 +48,12 @@ matrix:
4848
- mysql
4949
- postgresql
5050

51-
# In allow_failures
5251
- env:
5352
- JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
5453
services:
5554
- mysql
5655
- postgresql
5756

58-
allow_failures:
59-
- env:
60-
- JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
61-
6257
before_install:
6358
- echo "before_install"
6459
# set non-blocking IO on travis

‎asv_bench/asv.conf.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -122,5 +122,8 @@
122122
".*": "0409521665"
123123
},
124124
"regression_thresholds": {
125-
}
125+
},
126+
"build_command":
127+
["python setup.py build -j4",
128+
"PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"],
126129
}

‎asv_bench/benchmarks/binary_ops.py

+32
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import operator
2+
13
import numpy as np
24

35
from pandas import DataFrame, Series, date_range
@@ -9,6 +11,36 @@
911
import pandas.computation.expressions as expr
1012

1113

14+
class IntFrameWithScalar:
15+
params = [
16+
[np.float64, np.int64],
17+
[2, 3.0, np.int32(4), np.float64(5)],
18+
[
19+
operator.add,
20+
operator.sub,
21+
operator.mul,
22+
operator.truediv,
23+
operator.floordiv,
24+
operator.pow,
25+
operator.mod,
26+
operator.eq,
27+
operator.ne,
28+
operator.gt,
29+
operator.ge,
30+
operator.lt,
31+
operator.le,
32+
],
33+
]
34+
param_names = ["dtype", "scalar", "op"]
35+
36+
def setup(self, dtype, scalar, op):
37+
arr = np.random.randn(20000, 100)
38+
self.df = DataFrame(arr.astype(dtype))
39+
40+
def time_frame_op_with_scalar(self, dtype, scalar, op):
41+
op(self.df, scalar)
42+
43+
1244
class Ops:
1345

1446
params = [[True, False], ["default", 1]]

‎asv_bench/benchmarks/io/json.py

+24
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,30 @@ def peakmem_to_json_wide(self, orient, frame):
132132
df.to_json(self.fname, orient=orient)
133133

134134

135+
class ToJSONISO(BaseIO):
136+
fname = "__test__.json"
137+
params = [["split", "columns", "index", "values", "records"]]
138+
param_names = ["orient"]
139+
140+
def setup(self, orient):
141+
N = 10 ** 5
142+
index = date_range("20000101", periods=N, freq="H")
143+
timedeltas = timedelta_range(start=1, periods=N, freq="s")
144+
datetimes = date_range(start=1, periods=N, freq="s")
145+
self.df = DataFrame(
146+
{
147+
"td_1": timedeltas,
148+
"td_2": timedeltas,
149+
"ts_1": datetimes,
150+
"ts_2": datetimes,
151+
},
152+
index=index,
153+
)
154+
155+
def time_iso_format(self, orient):
156+
self.df.to_json(orient=orient, date_format="iso")
157+
158+
135159
class ToJSONLines(BaseIO):
136160

137161
fname = "__test__.json"

‎asv_bench/benchmarks/tslibs/offsets.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def setup(self, offset):
5959

6060
def time_on_offset(self, offset):
6161
for date in self.dates:
62-
offset.onOffset(date)
62+
offset.is_on_offset(date)
6363

6464

6565
class OffestDatetimeArithmetic:

‎ci/azure/posix.yml

+21-7
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,24 @@ jobs:
1919
ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
2020
CONDA_PY: "36"
2121
PATTERN: "not slow and not network"
22+
2223
py36_locale_slow_old_np:
2324
ENV_FILE: ci/deps/azure-36-locale_slow.yaml
2425
CONDA_PY: "36"
2526
PATTERN: "slow"
26-
LOCALE_OVERRIDE: "zh_CN.UTF-8"
27+
# pandas does not use the language (zh_CN), but should support diferent encodings (utf8)
28+
# we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
29+
LANG: "zh_CN.utf8"
30+
LC_ALL: "zh_CN.utf8"
2731
EXTRA_APT: "language-pack-zh-hans"
2832

2933
py36_locale:
3034
ENV_FILE: ci/deps/azure-36-locale.yaml
3135
CONDA_PY: "36"
3236
PATTERN: "not slow and not network"
33-
LOCALE_OVERRIDE: "it_IT.UTF-8"
37+
LANG: "it_IT.utf8"
38+
LC_ALL: "it_IT.utf8"
39+
EXTRA_APT: "language-pack-it"
3440

3541
py36_32bit:
3642
ENV_FILE: ci/deps/azure-36-32bit.yaml
@@ -42,7 +48,9 @@ jobs:
4248
ENV_FILE: ci/deps/azure-37-locale.yaml
4349
CONDA_PY: "37"
4450
PATTERN: "not slow and not network"
45-
LOCALE_OVERRIDE: "zh_CN.UTF-8"
51+
LANG: "zh_CN.utf8"
52+
LC_ALL: "zh_CN.utf8"
53+
EXTRA_APT: "language-pack-zh-hans"
4654

4755
py37_np_dev:
4856
ENV_FILE: ci/deps/azure-37-numpydev.yaml
@@ -54,10 +62,16 @@ jobs:
5462

5563
steps:
5664
- script: |
57-
if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386 $EXTRA_APT; fi
58-
echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
59-
echo "Creating Environment"
60-
ci/setup_env.sh
65+
if [ "$(uname)" == "Linux" ]; then
66+
sudo apt-get update
67+
sudo apt-get install -y libc6-dev-i386 $EXTRA_APT
68+
fi
69+
displayName: 'Install extra packages'
70+
71+
- script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
72+
displayName: 'Set conda path'
73+
74+
- script: ci/setup_env.sh
6175
displayName: 'Setup environment and build pandas'
6276

6377
- script: |

‎ci/azure/windows.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
- bash: |
3535
source activate pandas-dev
3636
conda list
37-
python setup.py build_ext -q -i
37+
python setup.py build_ext -q -i -j 4
3838
python -m pip install --no-build-isolation -e .
3939
displayName: 'Build'
4040

‎ci/code_checks.sh

+8
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,14 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
100100
cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime pandas/_libs/*.cpp
101101
RET=$(($RET + $?)) ; echo $MSG "DONE"
102102

103+
MSG='Check for use of not concatenated strings' ; echo $MSG
104+
if [[ "$GITHUB_ACTIONS" == "true" ]]; then
105+
$BASE_DIR/scripts/validate_string_concatenation.py --format="[error]{source_path}:{line_number}:{msg}" .
106+
else
107+
$BASE_DIR/scripts/validate_string_concatenation.py .
108+
fi
109+
RET=$(($RET + $?)) ; echo $MSG "DONE"
110+
103111
echo "isort --version-number"
104112
isort --version-number
105113

‎ci/deps/azure-36-locale_slow.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ dependencies:
1313
- pytest-azurepipelines
1414

1515
# pandas dependencies
16-
- beautifulsoup4==4.6.0
16+
- beautifulsoup4=4.6.0
1717
- bottleneck=1.2.*
1818
- lxml
1919
- matplotlib=2.2.2

‎ci/deps/azure-36-minimum_versions.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ dependencies:
1717
- beautifulsoup4=4.6.0
1818
- bottleneck=1.2.1
1919
- jinja2=2.8
20+
- numba=0.46.0
2021
- numexpr=2.6.2
2122
- numpy=1.13.3
2223
- openpyxl=2.5.7

‎ci/deps/azure-windows-36.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ dependencies:
1717
- bottleneck
1818
- fastparquet>=0.3.2
1919
- matplotlib=3.0.2
20+
- numba
2021
- numexpr
2122
- numpy=1.15.*
2223
- openpyxl

‎ci/deps/travis-37.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ dependencies:
2020
- pyarrow
2121
- pytz
2222
- s3fs
23+
- tabulate
2324
- pyreadstat
2425
- pip
2526
- pip:

‎ci/deps/travis-38.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ dependencies:
1717
- nomkl
1818
- pytz
1919
- pip
20+
- tabulate==0.8.3

‎ci/run_tests.sh

-11
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,6 @@
55
# https://github.com/pytest-dev/pytest/issues/1075
66
export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
77

8-
if [ -n "$LOCALE_OVERRIDE" ]; then
9-
export LC_ALL="$LOCALE_OVERRIDE"
10-
export LANG="$LOCALE_OVERRIDE"
11-
PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
12-
if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
13-
echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
14-
# TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
15-
# exit 1
16-
fi
17-
fi
18-
198
if [[ "not network" == *"$PATTERN"* ]]; then
209
export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
2110
fi

0 commit comments

Comments
 (0)