Skip to content

Commit 4ddb07a

Browse files
Fix Python3 errors for device tests
The Python3 migration didn't include fixes for local scripts in the device test tree. Fatal build and run Python errors fixed. The last update to xunitmerge is ~5 years ago, so it looks to be unsupported now. Use a local copy of the two components to allow patching to work with Python3. The serial test seems to send garbage chars (non-ASCII/etc.), so use a codepage 437 which supports all 255 chars. Fixes: esp8266#6660
1 parent 1e17ddd commit 4ddb07a

File tree

8 files changed

+180
-24
lines changed

8 files changed

+180
-24
lines changed

tests/device/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ ifneq ("$(NO_RUN)","1")
104104
endif
105105

106106
$(TEST_REPORT_XML): $(HARDWARE_DIR) virtualenv
107-
$(SILENT)$(BS_DIR)/virtualenv/bin/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
107+
$(SILENT)$(BS_DIR)/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
108108

109109
$(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv
110110
$(SILENT)$(BS_DIR)/virtualenv/bin/junit2html $< $@

tests/device/libraries/BSTest/requirements.txt

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,5 @@ junit-xml
33
MarkupSafe
44
pexpect
55
pyserial
6-
xunitmerge
76
junit2html
8-
poster
7+
poster3

tests/device/libraries/BSTest/runner.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -236,10 +236,10 @@ def request_env(self, key):
236236
def spawn_port(port_name, baudrate=115200):
237237
global ser
238238
ser = serial.serial_for_url(port_name, baudrate=baudrate)
239-
return fdpexpect.fdspawn(ser, 'wb', timeout=0)
239+
return fdpexpect.fdspawn(ser, 'wb', timeout=0, encoding='cp437')
240240

241241
def spawn_exec(name):
242-
return pexpect.spawn(name, timeout=0)
242+
return pexpect.spawn(name, timeout=0, encoding='cp437')
243243

244244
def run_tests(spawn, name, mocks, env_vars):
245245
tw = BSTestRunner(spawn, name, mocks, env_vars)
+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
from __future__ import unicode_literals, print_function
2+
from contextlib import contextmanager
3+
from xml.etree import ElementTree as etree
4+
from xml.sax.saxutils import quoteattr
5+
6+
import six
7+
8+
9+
CNAME_TAGS = ('system-out', 'skipped', 'error', 'failure')
10+
CNAME_PATTERN = '<![CDATA[{}]]>'
11+
TAG_PATTERN = '<{tag}{attrs}>{text}</{tag}>'
12+
13+
14+
@contextmanager
15+
def patch_etree_cname(etree):
16+
"""
17+
Patch ElementTree's _serialize_xml function so that it will
18+
write text as CDATA tag for tags tags defined in CNAME_TAGS.
19+
20+
>>> import re
21+
>>> from xml.etree import ElementTree
22+
>>> xml_string = '''
23+
... <testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
24+
... <testcase classname="some.class.Foo" name="test_system_out" time="0.001">
25+
... <system-out>Some output here</system-out>
26+
... </testcase>
27+
... <testcase classname="some.class.Foo" name="test_skipped" time="0.001">
28+
... <skipped type="unittest.case.SkipTest" message="Skipped">Skipped</skipped>
29+
... </testcase>
30+
... <testcase classname="some.class.Foo" name="test_error" time="0.001">
31+
... <error type="KeyError" message="Error here">Error here</error>
32+
... </testcase>
33+
... <testcase classname="some.class.Foo" name="test_failure" time="0.001">
34+
... <failure type="AssertionError" message="Failure here">Failure here</failure>
35+
... </testcase>
36+
... </testsuite>
37+
... '''
38+
>>> tree = ElementTree.fromstring(xml_string)
39+
>>> with patch_etree_cname(ElementTree):
40+
... saved = str(ElementTree.tostring(tree))
41+
>>> systemout = re.findall(r'(<system-out>.*?</system-out>)', saved)[0]
42+
>>> print(systemout)
43+
<system-out><![CDATA[Some output here]]></system-out>
44+
>>> skipped = re.findall(r'(<skipped.*?</skipped>)', saved)[0]
45+
>>> print(skipped)
46+
<skipped message="Skipped" type="unittest.case.SkipTest"><![CDATA[Skipped]]></skipped>
47+
>>> error = re.findall(r'(<error.*?</error>)', saved)[0]
48+
>>> print(error)
49+
<error message="Error here" type="KeyError"><![CDATA[Error here]]></error>
50+
>>> failure = re.findall(r'(<failure.*?</failure>)', saved)[0]
51+
>>> print(failure)
52+
<failure message="Failure here" type="AssertionError"><![CDATA[Failure here]]></failure>
53+
"""
54+
original_serialize = etree._serialize_xml
55+
56+
def _serialize_xml(write, elem, *args, **kwargs):
57+
if elem.tag in CNAME_TAGS:
58+
attrs = ' '.join(
59+
['{}={}'.format(k, quoteattr(v))
60+
for k, v in sorted(elem.attrib.items())]
61+
)
62+
attrs = ' ' + attrs if attrs else ''
63+
text = CNAME_PATTERN.format(elem.text)
64+
write(TAG_PATTERN.format(
65+
tag=elem.tag,
66+
attrs=attrs,
67+
text=text
68+
))
69+
else:
70+
original_serialize(write, elem, *args, **kwargs)
71+
72+
etree._serialize_xml = etree._serialize['xml'] = _serialize_xml
73+
74+
yield
75+
76+
etree._serialize_xml = etree._serialize['xml'] = original_serialize
77+
78+
79+
def merge_trees(*trees):
80+
"""
81+
Merge all given XUnit ElementTrees into a single ElementTree.
82+
This combines all of the children test-cases and also merges
83+
all of the metadata of how many tests were executed, etc.
84+
"""
85+
first_tree = trees[0]
86+
first_root = first_tree.getroot()
87+
88+
if len(trees) == 0:
89+
return first_tree
90+
91+
for tree in trees[1:]:
92+
root = tree.getroot()
93+
94+
# append children elements (testcases)
95+
first_root.extend(root.getchildren())
96+
97+
# combine root attributes which stores the number
98+
# of executed tests, skipped tests, etc
99+
for key, value in first_root.attrib.items():
100+
if not value.isdigit():
101+
continue
102+
combined = six.text_type(int(value) + int(root.attrib.get(key, '0')))
103+
first_root.set(key, combined)
104+
105+
return first_tree
106+
107+
108+
def merge_xunit(files, output, callback=None):
109+
"""
110+
Merge the given xunit xml files into a single output xml file.
111+
112+
If callback is not None, it will be called with the merged ElementTree
113+
before the output file is written (useful for applying other fixes to
114+
the merged file). This can either modify the element tree in place (and
115+
return None) or return a completely new ElementTree to be written.
116+
"""
117+
trees = []
118+
119+
for f in files:
120+
trees.append(etree.parse(f))
121+
122+
merged = merge_trees(*trees)
123+
124+
if callback is not None:
125+
result = callback(merged)
126+
if result is not None:
127+
merged = result
128+
129+
with patch_etree_cname(etree):
130+
merged.write(output, encoding='utf-8', xml_declaration=True)
+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/home/earle/Arduino/hardware/esp8266com/esp8266/tests/device/libraries/BSTest/virtualenv/bin/python3
2+
3+
from __future__ import unicode_literals, print_function
4+
import argparse
5+
from xmerge import merge_xunit
6+
7+
8+
parser = argparse.ArgumentParser(
9+
description='Utility for merging multiple XUnit xml reports '
10+
'into a single xml report.',
11+
)
12+
parser.add_argument(
13+
'report',
14+
nargs='+',
15+
type=argparse.FileType('r'),
16+
help='Path of XUnit xml report. Multiple can be provided.',
17+
)
18+
parser.add_argument(
19+
'output',
20+
help='Path where merged of XUnit will be saved.',
21+
)
22+
23+
24+
if __name__ == '__main__':
25+
args = parser.parse_args()
26+
merge_xunit(args.report, args.output)

tests/device/test_ClientContext/test_ClientContext.py

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env python3
2+
13
from mock_decorators import setup, teardown
24
from flask import Flask, request
35
from threading import Thread
@@ -21,7 +23,7 @@ def run():
2123
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
2224
for port in range(8266, 8285 + 1):
2325
try:
24-
print >>sys.stderr, 'trying port', port
26+
print ('trying port %d' %port, file=sys.stderr)
2527
server_address = ("0.0.0.0", port)
2628
sock.bind(server_address)
2729
sock.listen(1)
@@ -31,17 +33,17 @@ def run():
3133
print >>sys.stderr, 'busy'
3234
if not running:
3335
return
34-
print >>sys.stderr, 'starting up on %s port %s' % server_address
35-
print >>sys.stderr, 'waiting for connections'
36+
print ('starting up on %s port %s' % server_address, file=sys.stderr)
37+
print ( 'waiting for connections', file=sys.stderr)
3638
while running:
37-
print >>sys.stderr, 'loop'
39+
print ('loop', file=sys.stderr)
3840
readable, writable, errored = select.select([sock], [], [], 1.0)
3941
if readable:
4042
connection, client_address = sock.accept()
4143
try:
42-
print >>sys.stderr, 'client connected:', client_address
44+
print('client connected: %s' % str(client_address), file=sys.stderr)
4345
finally:
44-
print >>sys.stderr, 'close'
46+
print ('close', file=sys.stderr)
4547
connection.shutdown(socket.SHUT_RDWR)
4648
connection.close()
4749

@@ -54,7 +56,7 @@ def teardown_tcpsrv(e):
5456
global thread
5557
global running
5658

57-
print >>sys.stderr, 'closing'
59+
print ('closing', file=sys.stderr)
5860
running = False
5961
thread.join()
6062
return 0

tests/device/test_http_client/test_http_client.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from mock_decorators import setup, teardown
22
from flask import Flask, request, redirect
33
from threading import Thread
4-
import urllib2
4+
import urllib
55
import os
66
import ssl
77
import time
@@ -20,7 +20,7 @@ def shutdown():
2020
return 'Server shutting down...'
2121
@app.route("/", methods = ['GET', 'POST'])
2222
def root():
23-
print('Got data: ' + request.data);
23+
print('Got data: ' + request.data.decode());
2424
return 'hello!!!'
2525
@app.route("/data")
2626
def get_data():
@@ -48,7 +48,7 @@ def flaskThread():
4848

4949
@teardown('HTTP GET & POST requests')
5050
def teardown_http_get(e):
51-
response = urllib2.urlopen('http://localhost:8088/shutdown')
51+
response = urllib.request.urlopen('http://localhost:8088/shutdown')
5252
html = response.read()
5353
time.sleep(1) # avoid address in use error on macOS
5454

@@ -86,6 +86,6 @@ def teardown_http_get(e):
8686
ctx.check_hostname = False
8787
ctx.verify_mode = ssl.CERT_NONE
8888
p = os.path.dirname(os.path.abspath(__file__))
89-
response = urllib2.urlopen('https://localhost:8088/shutdown', context=ctx)
89+
response = urllib.request.urlopen('https://localhost:8088/shutdown', context=ctx)
9090
html = response.read()
9191

tests/device/test_http_server/test_http_server.py

+7-8
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
from collections import OrderedDict
22
from mock_decorators import setup, teardown
33
from threading import Thread
4-
from poster.encode import MultipartParam
5-
from poster.encode import multipart_encode
6-
from poster.streaminghttp import register_openers
7-
import urllib2
4+
from poster3.encode import MultipartParam
5+
from poster3.encode import multipart_encode
6+
from poster3.streaminghttp import register_openers
87
import urllib
98

109
def http_test(res, url, get=None, post=None):
@@ -13,8 +12,8 @@ def http_test(res, url, get=None, post=None):
1312
if get:
1413
url += '?' + urllib.urlencode(get)
1514
if post:
16-
post = urllib.urlencode(post)
17-
request = urllib2.urlopen(url, post, 2)
15+
post = urllib.parse.quote(post)
16+
request = urllib.request.urlopen(url, post, 2)
1817
response = request.read()
1918
except:
2019
return 1
@@ -60,8 +59,8 @@ def testRun():
6059
register_openers()
6160
p = MultipartParam("file", "0123456789abcdef", "test.txt", "text/plain; charset=utf8")
6261
datagen, headers = multipart_encode( [("var4", "val with spaces"), p] )
63-
request = urllib2.Request('http://etd.local/upload', datagen, headers)
64-
response = urllib2.urlopen(request, None, 2).read()
62+
request = urllib.request('http://etd.local/upload', datagen, headers)
63+
response = urllib.request.urlopen(request, None, 2).read()
6564
except:
6665
return 1
6766
if response != 'test.txt:16\nvar4 = val with spaces':

0 commit comments

Comments
 (0)