Context Navigation

← Previous Changeset
Next Changeset →

Changeset 334

Timestamp:

12/30/08 08:30:52 (3 years ago)

Author:

xi

Message:

Fixed str/bytes issues with Python 3 in _yaml.pyx.

Location:

Files:

: 6 modified
: 4 copied

ext/_yaml.h (modified) (1 diff)
ext/_yaml.pxd (modified) (1 diff)
ext/_yaml.pyx (modified) (51 diffs)
tests/data/invalid-base64-data-2.loader-error (copied) (copied from pyyaml/trunk/tests/data/invalid-base64-data.loader-error) (1 diff)
tests/data/invalid-python-bytes-2-py3.loader-error (copied) (copied from pyyaml/trunk/tests/data/invalid-base64-data.loader-error) (1 diff)
tests/data/invalid-python-bytes-py3.loader-error (copied) (copied from pyyaml/trunk/tests/data/invalid-base64-data.loader-error) (1 diff)
tests/lib/test_input_output.py (modified) (1 diff)
tests/lib3/test_input_output.py (copied) (copied from pyyaml/trunk/tests/lib/test_input_output.py) (5 diffs)
tests/lib3/test_yaml.py (modified) (1 diff)
tests/lib3/test_yaml_ext.py (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed

pyyaml/trunk/ext/_yaml.h

r331	r334
2	2	#include <yaml.h>
3	3
4		#if PY_MAJOR_VERSION >= 3
	4	#if PY_MAJOR_VERSION < 3
	5
	6	#define PyUnicode_FromString(s) PyUnicode_DecodeUTF8((s), strlen(s), 'strict')
	7
	8	#else
5	9
6	10	#define PyString_CheckExact PyBytes_CheckExact

pyyaml/trunk/ext/_yaml.pxd

r223	r334
10	10	int PyString_GET_SIZE(object o)
11	11	object PyString_FromStringAndSize(char *v, int l)
12		object PyUnicode_DecodeUTF8(char s, int s, char e)
	12	object PyUnicode_FromString(char *u)
	13	object PyUnicode_DecodeUTF8(char u, int s, char e)
13	14	object PyUnicode_AsUTF8String(object o)
	15	int PY_MAJOR_VERSION
14	16
15	17	ctypedef enum:

pyyaml/trunk/ext/_yaml.pyx

r333	r334
3	3
4	4	def get_version_string():
5		return yaml_get_version_string()
	5	cdef char *value
	6	value = yaml_get_version_string()
	7	if PY_MAJOR_VERSION < 3:
	8	return value
	9	else:
	10	return PyUnicode_FromString(value)
6	11
7	12	def get_version():
…	…
10	15	return (major, minor, patch)
11	16
12		#Mark = yaml.error.Mark
	17	Mark = yaml.error.Mark
13	18	YAMLError = yaml.error.YAMLError
14	19	ReaderError = yaml.reader.ReaderError
…	…
57	62	MappingNode = yaml.nodes.MappingNode
58	63
59		cdef class Mark:
60		cdef readonly object name
61		cdef readonly int index
62		cdef readonly int line
63		cdef readonly int column
64		cdef readonly buffer
65		cdef readonly pointer
66
67		def __init__(self, object name, int index, int line, int column,
68		object buffer, object pointer):
69		self.name = name
70		self.index = index
71		self.line = line
72		self.column = column
73		self.buffer = buffer
74		self.pointer = pointer
75
76		def get_snippet(self):
77		return None
78
79		def __str__(self):
80		where = " in \"%s\", line %d, column %d" \
81		% (self.name, self.line+1, self.column+1)
82		return where
83
	64	#cdef class Mark:
	65	# cdef readonly object name
	66	# cdef readonly int index
	67	# cdef readonly int line
	68	# cdef readonly int column
	69	# cdef readonly buffer
	70	# cdef readonly pointer
	71	#
	72	# def __init__(self, object name, int index, int line, int column,
	73	# object buffer, object pointer):
	74	# self.name = name
	75	# self.index = index
	76	# self.line = line
	77	# self.column = column
	78	# self.buffer = buffer
	79	# self.pointer = pointer
	80	#
	81	# def get_snippet(self):
	82	# return None
	83	#
	84	# def __str__(self):
	85	# where = " in \"%s\", line %d, column %d" \
	86	# % (self.name, self.line+1, self.column+1)
	87	# return where
	88	#
84	89	#class YAMLError(Exception):
85	90	# pass
…	…
271	276	self.stream_name = stream.name
272	277	except AttributeError:
273		self.stream_name = '<file>'
	278	if PY_MAJOR_VERSION < 3:
	279	self.stream_name = '<file>'
	280	else:
	281	self.stream_name = u'<file>'
274	282	self.stream_cache = None
275	283	self.stream_cache_len = 0
…	…
279	287	if PyUnicode_CheckExact(stream) != 0:
280	288	stream = PyUnicode_AsUTF8String(stream)
281		self.stream_name = '<unicode string>'
	289	if PY_MAJOR_VERSION < 3:
	290	self.stream_name = '<unicode string>'
	291	else:
	292	self.stream_name = u'<unicode string>'
282	293	self.unicode_source = 1
283	294	else:
284		self.stream_name = '<byte string>'
	295	if PY_MAJOR_VERSION < 3:
	296	self.stream_name = '<byte string>'
	297	else:
	298	self.stream_name = u'<byte string>'
285	299	if PyString_CheckExact(stream) == 0:
286		raise TypeError("a string or stream input is required")
	300	if PY_MAJOR_VERSION < 3:
	301	raise TypeError("a string or stream input is required")
	302	else:
	303	raise TypeError(u"a string or stream input is required")
287	304	self.stream = stream
288	305	yaml_parser_set_input_string(&self.parser, PyString_AS_STRING(stream), PyString_GET_SIZE(stream))
…	…
299	316	return MemoryError
300	317	elif self.parser.error == YAML_READER_ERROR:
301		return ReaderError(self.stream_name, self.parser.problem_offset,
302		self.parser.problem_value, '?', self.parser.problem)
	318	if PY_MAJOR_VERSION < 3:
	319	return ReaderError(self.stream_name, self.parser.problem_offset,
	320	self.parser.problem_value, '?', self.parser.problem)
	321	else:
	322	return ReaderError(self.stream_name, self.parser.problem_offset,
	323	self.parser.problem_value, u'?', PyUnicode_FromString(self.parser.problem))
303	324	elif self.parser.error == YAML_SCANNER_ERROR \
304	325	or self.parser.error == YAML_PARSER_ERROR:
…	…
315	336	self.parser.problem_mark.line,
316	337	self.parser.problem_mark.column, None, None)
	338	context = None
	339	if self.parser.context != NULL:
	340	if PY_MAJOR_VERSION < 3:
	341	context = self.parser.context
	342	else:
	343	context = PyUnicode_FromString(self.parser.context)
	344	if PY_MAJOR_VERSION < 3:
	345	problem = self.parser.problem
	346	else:
	347	problem = PyUnicode_FromString(self.parser.problem)
317	348	if self.parser.error == YAML_SCANNER_ERROR:
318		if self.parser.context != NULL:
319		return ScannerError(self.parser.context, context_mark,
320		self.parser.problem, problem_mark)
321		else:
322		return ScannerError(None, None,
323		self.parser.problem, problem_mark)
324		else:
325		if self.parser.context != NULL:
326		return ParserError(self.parser.context, context_mark,
327		self.parser.problem, problem_mark)
328		else:
329		return ParserError(None, None,
330		self.parser.problem, problem_mark)
331		raise ValueError("no parser error")
	349	return ScannerError(context, context_mark, problem, problem_mark)
	350	else:
	351	return ParserError(context, context_mark, problem, problem_mark)
	352	if PY_MAJOR_VERSION < 3:
	353	raise ValueError("no parser error")
	354	else:
	355	raise ValueError(u"no parser error")
332	356
333	357	def raw_scan(self):
…	…
388	412	start_mark, end_mark)
389	413	elif token.type == YAML_TAG_DIRECTIVE_TOKEN:
390		handle = PyUnicode_DecodeUTF8(token.data.tag_directive.handle,
391		strlen(token.data.tag_directive.handle), 'strict')
392		prefix = PyUnicode_DecodeUTF8(token.data.tag_directive.prefix,
393		strlen(token.data.tag_directive.prefix), 'strict')
	414	handle = PyUnicode_FromString(token.data.tag_directive.handle)
	415	prefix = PyUnicode_FromString(token.data.tag_directive.prefix)
394	416	return DirectiveToken(u"TAG", (handle, prefix),
395	417	start_mark, end_mark)
…	…
421	443	return ValueToken(start_mark, end_mark)
422	444	elif token.type == YAML_ALIAS_TOKEN:
423		value = PyUnicode_DecodeUTF8(token.data.alias.value,
424		strlen(token.data.alias.value), 'strict')
	445	value = PyUnicode_FromString(token.data.alias.value)
425	446	return AliasToken(value, start_mark, end_mark)
426	447	elif token.type == YAML_ANCHOR_TOKEN:
427		value = PyUnicode_DecodeUTF8(token.data.anchor.value,
428		strlen(token.data.anchor.value), 'strict')
	448	value = PyUnicode_FromString(token.data.anchor.value)
429	449	return AnchorToken(value, start_mark, end_mark)
430	450	elif token.type == YAML_TAG_TOKEN:
431		handle = PyUnicode_DecodeUTF8(token.data.tag.handle,
432		strlen(token.data.tag.handle), 'strict')
433		suffix = PyUnicode_DecodeUTF8(token.data.tag.suffix,
434		strlen(token.data.tag.suffix), 'strict')
	451	handle = PyUnicode_FromString(token.data.tag.handle)
	452	suffix = PyUnicode_FromString(token.data.tag.suffix)
435	453	if not handle:
436	454	handle = None
…	…
443	461	if token.data.scalar.style == YAML_PLAIN_SCALAR_STYLE:
444	462	plain = True
445		style = ''
	463	style = u''
446	464	elif token.data.scalar.style == YAML_SINGLE_QUOTED_SCALAR_STYLE:
447		style = '\''
	465	style = u'\''
448	466	elif token.data.scalar.style == YAML_DOUBLE_QUOTED_SCALAR_STYLE:
449		style = '"'
	467	style = u'"'
450	468	elif token.data.scalar.style == YAML_LITERAL_SCALAR_STYLE:
451		style = '\|'
	469	style = u'\|'
452	470	elif token.data.scalar.style == YAML_FOLDED_SCALAR_STYLE:
453		style = '>'
	471	style = u'>'
454	472	return ScalarToken(value, plain,
455	473	start_mark, end_mark, style)
456	474	else:
457		raise ValueError("unknown token type")
	475	if PY_MAJOR_VERSION < 3:
	476	raise ValueError("unknown token type")
	477	else:
	478	raise ValueError(u"unknown token type")
458	479
459	480	def get_token(self):
…	…
527	548	if event.data.stream_start.encoding == YAML_UTF8_ENCODING:
528	549	if self.unicode_source == 0:
529		encoding = "utf-8"
	550	encoding = u"utf-8"
530	551	elif event.data.stream_start.encoding == YAML_UTF16LE_ENCODING:
531		encoding = "utf-16-le"
	552	encoding = u"utf-16-le"
532	553	elif event.data.stream_start.encoding == YAML_UTF16BE_ENCODING:
533		encoding = "utf-16-be"
	554	encoding = u"utf-16-be"
534	555	return StreamStartEvent(start_mark, end_mark, encoding)
535	556	elif event.type == YAML_STREAM_END_EVENT:
536	557	return StreamEndEvent(start_mark, end_mark)
537
538	558	elif event.type == YAML_DOCUMENT_START_EVENT:
539	559	explicit = False
…	…
549	569	tag_directive = event.data.document_start.tag_directives.start
550	570	while tag_directive != event.data.document_start.tag_directives.end:
551		handle = PyUnicode_DecodeUTF8(tag_directive.handle,
552		strlen(tag_directive.handle), 'strict')
553		prefix = PyUnicode_DecodeUTF8(tag_directive.prefix,
554		strlen(tag_directive.prefix), 'strict')
	571	handle = PyUnicode_FromString(tag_directive.handle)
	572	prefix = PyUnicode_FromString(tag_directive.prefix)
555	573	tags[handle] = prefix
556	574	tag_directive = tag_directive+1
…	…
563	581	return DocumentEndEvent(start_mark, end_mark, explicit)
564	582	elif event.type == YAML_ALIAS_EVENT:
565		anchor = PyUnicode_DecodeUTF8(event.data.alias.anchor,
566		strlen(event.data.alias.anchor), 'strict')
	583	anchor = PyUnicode_FromString(event.data.alias.anchor)
567	584	return AliasEvent(anchor, start_mark, end_mark)
568	585	elif event.type == YAML_SCALAR_EVENT:
569	586	anchor = None
570	587	if event.data.scalar.anchor != NULL:
571		anchor = PyUnicode_DecodeUTF8(event.data.scalar.anchor,
572		strlen(event.data.scalar.anchor), 'strict')
	588	anchor = PyUnicode_FromString(event.data.scalar.anchor)
573	589	tag = None
574	590	if event.data.scalar.tag != NULL:
575		tag = PyUnicode_DecodeUTF8(event.data.scalar.tag,
576		strlen(event.data.scalar.tag), 'strict')
	591	tag = PyUnicode_FromString(event.data.scalar.tag)
577	592	value = PyUnicode_DecodeUTF8(event.data.scalar.value,
578	593	event.data.scalar.length, 'strict')
…	…
585	600	style = None
586	601	if event.data.scalar.style == YAML_PLAIN_SCALAR_STYLE:
587		style = ''
	602	style = u''
588	603	elif event.data.scalar.style == YAML_SINGLE_QUOTED_SCALAR_STYLE:
589		style = '\''
	604	style = u'\''
590	605	elif event.data.scalar.style == YAML_DOUBLE_QUOTED_SCALAR_STYLE:
591		style = '"'
	606	style = u'"'
592	607	elif event.data.scalar.style == YAML_LITERAL_SCALAR_STYLE:
593		style = '\|'
	608	style = u'\|'
594	609	elif event.data.scalar.style == YAML_FOLDED_SCALAR_STYLE:
595		style = '>'
	610	style = u'>'
596	611	return ScalarEvent(anchor, tag,
597	612	(plain_implicit, quoted_implicit),
…	…
600	615	anchor = None
601	616	if event.data.sequence_start.anchor != NULL:
602		anchor = PyUnicode_DecodeUTF8(event.data.sequence_start.anchor,
603		strlen(event.data.sequence_start.anchor), 'strict')
	617	anchor = PyUnicode_FromString(event.data.sequence_start.anchor)
604	618	tag = None
605	619	if event.data.sequence_start.tag != NULL:
606		tag = PyUnicode_DecodeUTF8(event.data.sequence_start.tag,
607		strlen(event.data.sequence_start.tag), 'strict')
	620	tag = PyUnicode_FromString(event.data.sequence_start.tag)
608	621	implicit = False
609	622	if event.data.sequence_start.implicit == 1:
…	…
619	632	anchor = None
620	633	if event.data.mapping_start.anchor != NULL:
621		anchor = PyUnicode_DecodeUTF8(event.data.mapping_start.anchor,
622		strlen(event.data.mapping_start.anchor), 'strict')
	634	anchor = PyUnicode_FromString(event.data.mapping_start.anchor)
623	635	tag = None
624	636	if event.data.mapping_start.tag != NULL:
625		tag = PyUnicode_DecodeUTF8(event.data.mapping_start.tag,
626		strlen(event.data.mapping_start.tag), 'strict')
	637	tag = PyUnicode_FromString(event.data.mapping_start.tag)
627	638	implicit = False
628	639	if event.data.mapping_start.implicit == 1:
…	…
639	650	elif event.type == YAML_MAPPING_END_EVENT:
640	651	return MappingEndEvent(start_mark, end_mark)
641
642	652	else:
643		raise ValueError("unknown token type")
	653	if PY_MAJOR_VERSION < 3:
	654	raise ValueError("unknown event type")
	655	else:
	656	raise ValueError(u"unknown event type")
644	657
645	658	def get_event(self):
…	…
697	710	self.parsed_event.start_mark.column,
698	711	None, None)
699		raise ComposerError("expected a single document in the stream",
700		document.start_mark, "but found another document", mark)
	712	if PY_MAJOR_VERSION < 3:
	713	raise ComposerError("expected a single document in the stream",
	714	document.start_mark, "but found another document", mark)
	715	else:
	716	raise ComposerError(u"expected a single document in the stream",
	717	document.start_mark, u"but found another document", mark)
701	718	return document
702	719
…	…
712	729	self._parse_next_event()
713	730	if self.parsed_event.type == YAML_ALIAS_EVENT:
714		anchor = PyUnicode_DecodeUTF8(self.parsed_event.data.alias.anchor,
715		strlen(self.parsed_event.data.alias.anchor), 'strict')
	731	anchor = PyUnicode_FromString(self.parsed_event.data.alias.anchor)
716	732	if anchor not in self.anchors:
717	733	mark = Mark(self.stream_name,
…	…
720	736	self.parsed_event.start_mark.column,
721	737	None, None)
722		raise ComposerError(None, None, "found undefined alias", mark)
	738	if PY_MAJOR_VERSION < 3:
	739	raise ComposerError(None, None, "found undefined alias", mark)
	740	else:
	741	raise ComposerError(None, None, u"found undefined alias", mark)
723	742	yaml_event_delete(&self.parsed_event)
724	743	return self.anchors[anchor]
…	…
726	745	if self.parsed_event.type == YAML_SCALAR_EVENT \
727	746	and self.parsed_event.data.scalar.anchor != NULL:
728		anchor = PyUnicode_DecodeUTF8(self.parsed_event.data.scalar.anchor,
729		strlen(self.parsed_event.data.scalar.anchor), 'strict')
	747	anchor = PyUnicode_FromString(self.parsed_event.data.scalar.anchor)
730	748	elif self.parsed_event.type == YAML_SEQUENCE_START_EVENT \
731	749	and self.parsed_event.data.sequence_start.anchor != NULL:
732		anchor = PyUnicode_DecodeUTF8(self.parsed_event.data.sequence_start.anchor,
733		strlen(self.parsed_event.data.sequence_start.anchor), 'strict')
	750	anchor = PyUnicode_FromString(self.parsed_event.data.sequence_start.anchor)
734	751	elif self.parsed_event.type == YAML_MAPPING_START_EVENT \
735	752	and self.parsed_event.data.mapping_start.anchor != NULL:
736		anchor = PyUnicode_DecodeUTF8(self.parsed_event.data.mapping_start.anchor,
737		strlen(self.parsed_event.data.mapping_start.anchor), 'strict')
	753	anchor = PyUnicode_FromString(self.parsed_event.data.mapping_start.anchor)
738	754	if anchor is not None:
739	755	if anchor in self.anchors:
…	…
743	759	self.parsed_event.start_mark.column,
744	760	None, None)
745		raise ComposerError("found duplicate anchor; first occurence",
746		self.anchors[anchor].start_mark, "second occurence", mark)
	761	if PY_MAJOR_VERSION < 3:
	762	raise ComposerError("found duplicate anchor; first occurence",
	763	self.anchors[anchor].start_mark, "second occurence", mark)
	764	else:
	765	raise ComposerError(u"found duplicate anchor; first occurence",
	766	self.anchors[anchor].start_mark, u"second occurence", mark)
747	767	self.descend_resolver(parent, index)
748	768	if self.parsed_event.type == YAML_SCALAR_EVENT:
…	…
779	799	tag = self.resolve(ScalarNode, value, (plain_implicit, quoted_implicit))
780	800	else:
781		tag = PyUnicode_DecodeUTF8(self.parsed_event.data.scalar.tag,
782		strlen(self.parsed_event.data.scalar.tag), 'strict')
	801	tag = PyUnicode_FromString(self.parsed_event.data.scalar.tag)
783	802	style = None
784	803	if self.parsed_event.data.scalar.style == YAML_PLAIN_SCALAR_STYLE:
785		style = ''
	804	style = u''
786	805	elif self.parsed_event.data.scalar.style == YAML_SINGLE_QUOTED_SCALAR_STYLE:
787		style = '\''
	806	style = u'\''
788	807	elif self.parsed_event.data.scalar.style == YAML_DOUBLE_QUOTED_SCALAR_STYLE:
789		style = '"'
	808	style = u'"'
790	809	elif self.parsed_event.data.scalar.style == YAML_LITERAL_SCALAR_STYLE:
791		style = '\|'
	810	style = u'\|'
792	811	elif self.parsed_event.data.scalar.style == YAML_FOLDED_SCALAR_STYLE:
793		style = '>'
	812	style = u'>'
794	813	node = ScalarNode(tag, value, start_mark, end_mark, style)
795	814	if anchor is not None:
…	…
813	832	tag = self.resolve(SequenceNode, None, implicit)
814	833	else:
815		tag = PyUnicode_DecodeUTF8(self.parsed_event.data.sequence_start.tag,
816		strlen(self.parsed_event.data.sequence_start.tag), 'strict')
	834	tag = PyUnicode_FromString(self.parsed_event.data.sequence_start.tag)
817	835	flow_style = None
818	836	if self.parsed_event.data.sequence_start.style == YAML_FLOW_SEQUENCE_STYLE:
…	…
853	871	tag = self.resolve(MappingNode, None, implicit)
854	872	else:
855		tag = PyUnicode_DecodeUTF8(self.parsed_event.data.mapping_start.tag,
856		strlen(self.parsed_event.data.mapping_start.tag), 'strict')
	873	tag = PyUnicode_FromString(self.parsed_event.data.mapping_start.tag)
857	874	flow_style = None
858	875	if self.parsed_event.data.mapping_start.style == YAML_FLOW_MAPPING_STYLE:
…	…
895	912	parser.unicode_source = 1
896	913	if PyString_CheckExact(value) == 0:
897		raise TypeError("a string value is expected")
	914	if PY_MAJOR_VERSION < 3:
	915	raise TypeError("a string value is expected")
	916	else:
	917	raise TypeError(u"a string value is expected")
898	918	parser.stream_cache = value
899	919	parser.stream_cache_pos = 0
…	…
977	997	return MemoryError
978	998	elif self.emitter.error == YAML_EMITTER_ERROR:
979		return EmitterError(self.emitter.problem)
980		raise ValueError("no emitter error")
	999	if PY_MAJOR_VERSION < 3:
	1000	problem = self.emitter.problem
	1001	else:
	1002	problem = PyUnicode_FromString(self.emitter.problem)
	1003	return EmitterError(problem)
	1004	if PY_MAJOR_VERSION < 3:
	1005	raise ValueError("no emitter error")
	1006	else:
	1007	raise ValueError(u"no emitter error")
981	1008
982	1009	cdef int _object_to_event(self, object event_object, yaml_event_t *event) except 0:
…	…
1000	1027	if event_class is StreamStartEvent:
1001	1028	encoding = YAML_UTF8_ENCODING
1002		if event_object.encoding == 'utf-16-le':
	1029	if event_object.encoding == u'utf-16-le' or event_object.encoding == 'utf-16-le':
1003	1030	encoding = YAML_UTF16LE_ENCODING
1004		elif event_object.encoding == 'utf-16-be':
	1031	elif event_object.encoding == u'utf-16-be' or event_object.encoding == 'utf-16-be':
1005	1032	encoding = YAML_UTF16BE_ENCODING
1006	1033	if event_object.encoding is None:
…	…
1021	1048	if event_object.tags:
1022	1049	if len(event_object.tags) > 128:
1023		raise ValueError("too many tags")
	1050	if PY_MAJOR_VERSION < 3:
	1051	raise ValueError("too many tags")
	1052	else:
	1053	raise ValueError(u"too many tags")
1024	1054	tag_directives_start = tag_directives_value
1025	1055	tag_directives_end = tag_directives_value
…	…
1031	1061	cache.append(handle)
1032	1062	if not PyString_CheckExact(handle):
1033		raise TypeError("tag handle must be a string")
	1063	if PY_MAJOR_VERSION < 3:
	1064	raise TypeError("tag handle must be a string")
	1065	else:
	1066	raise TypeError(u"tag handle must be a string")
1034	1067	tag_directives_end.handle = PyString_AS_STRING(handle)
1035	1068	if PyUnicode_CheckExact(prefix):
…	…
1037	1070	cache.append(prefix)
1038	1071	if not PyString_CheckExact(prefix):
1039		raise TypeError("tag prefix must be a string")
	1072	if PY_MAJOR_VERSION < 3:
	1073	raise TypeError("tag prefix must be a string")
	1074	else:
	1075	raise TypeError(u"tag prefix must be a string")
1040	1076	tag_directives_end.prefix = PyString_AS_STRING(prefix)
1041	1077	tag_directives_end = tag_directives_end+1
…	…
1057	1093	anchor_object = PyUnicode_AsUTF8String(anchor_object)
1058	1094	if not PyString_CheckExact(anchor_object):
1059		raise TypeError("anchor must be a string")
	1095	if PY_MAJOR_VERSION < 3:
	1096	raise TypeError("anchor must be a string")
	1097	else:
	1098	raise TypeError(u"anchor must be a string")
1060	1099	anchor = PyString_AS_STRING(anchor_object)
1061	1100	if yaml_alias_event_initialize(event, anchor) == 0:
…	…
1068	1107	anchor_object = PyUnicode_AsUTF8String(anchor_object)
1069	1108	if not PyString_CheckExact(anchor_object):
1070		raise TypeError("anchor must be a string")
	1109	if PY_MAJOR_VERSION < 3:
	1110	raise TypeError("anchor must be a string")
	1111	else:
	1112	raise TypeError(u"anchor must be a string")
1071	1113	anchor = PyString_AS_STRING(anchor_object)
1072	1114	tag = NULL
…	…
1076	1118	tag_object = PyUnicode_AsUTF8String(tag_object)
1077	1119	if not PyString_CheckExact(tag_object):
1078		raise TypeError("tag must be a string")
	1120	if PY_MAJOR_VERSION < 3:
	1121	raise TypeError("tag must be a string")
	1122	else:
	1123	raise TypeError(u"tag must be a string")
1079	1124	tag = PyString_AS_STRING(tag_object)
1080	1125	value_object = event_object.value
…	…
1082	1127	value_object = PyUnicode_AsUTF8String(value_object)
1083	1128	if not PyString_CheckExact(value_object):
1084		raise TypeError("value must be a string")
	1129	if PY_MAJOR_VERSION < 3:
	1130	raise TypeError("value must be a string")
	1131	else:
	1132	raise TypeError(u"value must be a string")
1085	1133	value = PyString_AS_STRING(value_object)
1086	1134	length = PyString_GET_SIZE(value_object)
…	…
1092	1140	style_object = event_object.style
1093	1141	scalar_style = YAML_PLAIN_SCALAR_STYLE
1094		if style_object == "'":
	1142	if style_object == "'" or style_object == u"'":
1095	1143	scalar_style = YAML_SINGLE_QUOTED_SCALAR_STYLE
1096		elif style_object == "\"":
	1144	elif style_object == "\"" or style_object == u"\"":
1097	1145	scalar_style = YAML_DOUBLE_QUOTED_SCALAR_STYLE
1098		elif style_object == "\|":
	1146	elif style_object == "\|" or style_object == u"\|":
1099	1147	scalar_style = YAML_LITERAL_SCALAR_STYLE
1100		elif style_object == ">":
	1148	elif style_object == ">" or style_object == u">":
1101	1149	scalar_style = YAML_FOLDED_SCALAR_STYLE
1102	1150	if yaml_scalar_event_initialize(event, anchor, tag, value, length,
…	…
1110	1158	anchor_object = PyUnicode_AsUTF8String(anchor_object)
1111	1159	if not PyString_CheckExact(anchor_object):
1112		raise TypeError("anchor must be a string")
	1160	if PY_MAJOR_VERSION < 3:
	1161	raise TypeError("anchor must be a string")
	1162	else:
	1163	raise TypeError(u"anchor must be a string")
1113	1164	anchor = PyString_AS_STRING(anchor_object)
1114	1165	tag = NULL
…	…
1118	1169	tag_object = PyUnicode_AsUTF8String(tag_object)
1119	1170	if not PyString_CheckExact(tag_object):
1120		raise TypeError("tag must be a string")
	1171	if PY_MAJOR_VERSION < 3:
	1172	raise TypeError("tag must be a string")
	1173	else:
	1174	raise TypeError(u"tag must be a string")
1121	1175	tag = PyString_AS_STRING(tag_object)
1122	1176	implicit = 0
…	…
1136	1190	anchor_object = PyUnicode_AsUTF8String(anchor_object)
1137	1191	if not PyString_CheckExact(anchor_object):
1138		raise TypeError("anchor must be a string")
	1192	if PY_MAJOR_VERSION < 3:
	1193	raise TypeError("anchor must be a string")
	1194	else:
	1195	raise TypeError(u"anchor must be a string")
1139	1196	anchor = PyString_AS_STRING(anchor_object)
1140	1197	tag = NULL
…	…
1144	1201	tag_object = PyUnicode_AsUTF8String(tag_object)
1145	1202	if not PyString_CheckExact(tag_object):
1146		raise TypeError("tag must be a string")
	1203	if PY_MAJOR_VERSION < 3:
	1204	raise TypeError("tag must be a string")
	1205	else:
	1206	raise TypeError(u"tag must be a string")
1147	1207	tag = PyString_AS_STRING(tag_object)
1148	1208	implicit = 0
…	…
1160	1220	yaml_mapping_end_event_initialize(event)
1161	1221	else:
1162		raise TypeError("invalid event %s" % event_object)
	1222	if PY_MAJOR_VERSION < 3:
	1223	raise TypeError("invalid event %s" % event_object)
	1224	else:
	1225	raise TypeError(u"invalid event %s" % event_object)
1163	1226	return 1
1164	1227
…	…
1174	1237	cdef yaml_encoding_t encoding
1175	1238	if self.closed == -1:
1176		if self.use_encoding == 'utf-16-le':
	1239	if self.use_encoding == u'utf-16-le' or self.use_encoding == 'utf-16-le':
1177	1240	encoding = YAML_UTF16LE_ENCODING
1178		elif self.use_encoding == 'utf-16-be':
	1241	elif self.use_encoding == u'utf-16-be' or self.use_encoding == 'utf-16-be':
1179	1242	encoding = YAML_UTF16BE_ENCODING
1180	1243	else:
…	…
1190	1253	self.closed = 0
1191	1254	elif self.closed == 1:
1192		raise SerializerError("serializer is closed")
	1255	if PY_MAJOR_VERSION < 3:
	1256	raise SerializerError("serializer is closed")
	1257	else:
	1258	raise SerializerError(u"serializer is closed")
1193	1259	else:
1194		raise SerializerError("serializer is already opened")
	1260	if PY_MAJOR_VERSION < 3:
	1261	raise SerializerError("serializer is already opened")
	1262	else:
	1263	raise SerializerError(u"serializer is already opened")
1195	1264
1196	1265	def close(self):
1197	1266	cdef yaml_event_t event
1198	1267	if self.closed == -1:
1199		raise SerializerError("serializer is not opened")
	1268	if PY_MAJOR_VERSION < 3:
	1269	raise SerializerError("serializer is not opened")
	1270	else:
	1271	raise SerializerError(u"serializer is not opened")
1200	1272	elif self.closed == 0:
1201	1273	yaml_stream_end_event_initialize(&event)
…	…
1213	1285	cdef yaml_tag_directive_t *tag_directives_end
1214	1286	if self.closed == -1:
1215		raise SerializerError("serializer is not opened")
	1287	if PY_MAJOR_VERSION < 3:
	1288	raise SerializerError("serializer is not opened")
	1289	else:
	1290	raise SerializerError(u"serializer is not opened")
1216	1291	elif self.closed == 1:
1217		raise SerializerError("serializer is closed")
	1292	if PY_MAJOR_VERSION < 3:
	1293	raise SerializerError("serializer is closed")
	1294	else:
	1295	raise SerializerError(u"serializer is closed")
1218	1296	cache = []
1219	1297	version_directive = NULL
…	…
1226	1304	if self.use_tags:
1227	1305	if len(self.use_tags) > 128:
1228		raise ValueError("too many tags")
	1306	if PY_MAJOR_VERSION < 3:
	1307	raise ValueError("too many tags")
	1308	else:
	1309	raise ValueError(u"too many tags")
1229	1310	tag_directives_start = tag_directives_value
1230	1311	tag_directives_end = tag_directives_value
…	…
1235	1316	cache.append(handle)
1236	1317	if not PyString_CheckExact(handle):
1237		raise TypeError("tag handle must be a string")
	1318	if PY_MAJOR_VERSION < 3:
	1319	raise TypeError("tag handle must be a string")
	1320	else:
	1321	raise TypeError(u"tag handle must be a string")
1238	1322	tag_directives_end.handle = PyString_AS_STRING(handle)
1239	1323	if PyUnicode_CheckExact(prefix):
…	…
1241	1325	cache.append(prefix)
1242	1326	if not PyString_CheckExact(prefix):
1243		raise TypeError("tag prefix must be a string")
	1327	if PY_MAJOR_VERSION < 3:
	1328	raise TypeError("tag prefix must be a string")
	1329	else:
	1330	raise TypeError(u"tag prefix must be a string")
1244	1331	tag_directives_end.prefix = PyString_AS_STRING(prefix)
1245	1332	tag_directives_end = tag_directives_end+1
…	…
1318	1405	tag_object = PyUnicode_AsUTF8String(tag_object)
1319	1406	if not PyString_CheckExact(tag_object):
1320		raise TypeError("tag must be a string")
	1407	if PY_MAJOR_VERSION < 3:
	1408	raise TypeError("tag must be a string")
	1409	else:
	1410	raise TypeError(u"tag must be a string")
1321	1411	tag = PyString_AS_STRING(tag_object)
1322	1412	value_object = node.value
…	…
1324	1414	value_object = PyUnicode_AsUTF8String(value_object)
1325	1415	if not PyString_CheckExact(value_object):
1326		raise TypeError("value must be a string")
	1416	if PY_MAJOR_VERSION < 3:
	1417	raise TypeError("value must be a string")
	1418	else:
	1419	raise TypeError(u"value must be a string")
1327	1420	value = PyString_AS_STRING(value_object)
1328	1421	length = PyString_GET_SIZE(value_object)
1329	1422	style_object = node.style
1330	1423	scalar_style = YAML_PLAIN_SCALAR_STYLE
1331		if style_object == "'":
	1424	if style_object == "'" or style_object == u"'":
1332	1425	scalar_style = YAML_SINGLE_QUOTED_SCALAR_STYLE
1333		elif style_object == "\"":
	1426	elif style_object == "\"" or style_object == u"\"":
1334	1427	scalar_style = YAML_DOUBLE_QUOTED_SCALAR_STYLE
1335		elif style_object == "\|":
	1428	elif style_object == "\|" or style_object == u"\|":
1336	1429	scalar_style = YAML_LITERAL_SCALAR_STYLE
1337		elif style_object == ">":
	1430	elif style_object == ">" or style_object == u">":
1338	1431	scalar_style = YAML_FOLDED_SCALAR_STYLE
1339	1432	if yaml_scalar_event_initialize(&event, anchor, tag, value, length,
…	…
1353	1446	tag_object = PyUnicode_AsUTF8String(tag_object)
1354	1447	if not PyString_CheckExact(tag_object):
1355		raise TypeError("tag must be a string")
	1448	if PY_MAJOR_VERSION < 3:
	1449	raise TypeError("tag must be a string")
	1450	else:
	1451	raise TypeError(u"tag must be a string")
1356	1452	tag = PyString_AS_STRING(tag_object)
1357	1453	sequence_style = YAML_BLOCK_SEQUENCE_STYLE
…	…
1382	1478	tag_object = PyUnicode_AsUTF8String(tag_object)
1383	1479	if not PyString_CheckExact(tag_object):
1384		raise TypeError("tag must be a string")
	1480	if PY_MAJOR_VERSION < 3:
	1481	raise TypeError("tag must be a string")
	1482	else:
	1483	raise TypeError(u"tag must be a string")
1385	1484	tag = PyString_AS_STRING(tag_object)
1386	1485	mapping_style = YAML_BLOCK_MAPPING_STYLE

pyyaml/trunk/tests/data/invalid-base64-data-2.loader-error

r140	r334
1	1	--- !!binary
2		~~binary data encoded in base64 should be here.~~
	2	ÐŽÐ²ÐŸÐžÑÐœÑÐµ ÐŽÐ°ÐœÐœÑÐµ Ð² base64

pyyaml/trunk/tests/data/invalid-python-bytes-2-py3.loader-error

r140	r334
1		--- !!~~binary~~
2		~~binary data encoded in base64 should be here.~~
	1	--- !!python/bytes
	2	ÐŽÐ²ÐŸÐžÑÐœÑÐµ ÐŽÐ°ÐœÐœÑÐµ Ð² base64

pyyaml/trunk/tests/data/invalid-python-bytes-py3.loader-error

r140	r334
1		--- !!~~binary~~
	1	--- !!python/bytes
2	2	binary data encoded in base64 should be here.

pyyaml/trunk/tests/lib/test_input_output.py

r333	r334
54	54	data = open(unicode_filename, 'rb').read().decode('utf-8')
55	55	value = ' '.join(data.split())
56		for ~~encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le'~~]:
57		~~for allow_unicode in [False, True]:~~
58		~~data1 = yaml.dump(value, allow_unicode=allow_unicode)~~
	56	for allow_unicode in [False, True]:
	57	data1 = yaml.dump(value, allow_unicode=allow_unicode)
	58	for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']:
59	59	stream = StringIO.StringIO()
60	60	yaml.dump(value, _unicode_open(stream, 'utf-8'), encoding=encoding, allow_unicode=allow_unicode)

pyyaml/trunk/tests/lib3/test_input_output.py

r333	r334
1	1
2	2	import yaml
3		import codecs, StringIO
4
5		def _unicode_open(file, encoding, errors='strict'):
6		info = codecs.lookup(encoding)
7		srw = codecs.StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
8		srw.encoding = encoding
9		return srw
	3	import codecs, io
10	4
11	5	def test_unicode_input(unicode_filename, verbose=False):
12	6	data = open(unicode_filename, 'rb').read().decode('utf-8')
13	7	value = ' '.join(data.split())
14		output = yaml.load(~~_unicode_open(StringIO.StringIO(data.encode('utf-8')), 'utf-8')~~)
	8	output = yaml.load(data)
15	9	assert output == value, (output, value)
16		for input in [data, data.encode('utf-8'),
	10	output = yaml.load(io.StringIO(data))
	11	assert output == value, (output, value)
	12	for input in [data.encode('utf-8'),
17	13	codecs.BOM_UTF8+data.encode('utf-8'),
18	14	codecs.BOM_UTF16_BE+data.encode('utf-16-be'),
19	15	codecs.BOM_UTF16_LE+data.encode('utf-16-le')]:
20	16	if verbose:
21		print ~~"INPUT:", repr(input[:10]), "..."~~
	17	print("INPUT:", repr(input[:10]), "...")
22	18	output = yaml.load(input)
23	19	assert output == value, (output, value)
24		output = yaml.load(~~StringIO.String~~IO(input))
	20	output = yaml.load(io.BytesIO(input))
25	21	assert output == value, (output, value)
26	22
…	…
33	29	codecs.BOM_UTF8+data.encode('utf-16-be'),
34	30	codecs.BOM_UTF16_BE+data.encode('utf-16-le'),
35		codecs.BOM_UTF16_LE+data.encode('utf-8')+'!']:
	31	codecs.BOM_UTF16_LE+data.encode('utf-8')+b'!']:
36	32	try:
37	33	yaml.load(input)
38		except yaml.YAMLError, exc:
	34	except yaml.YAMLError as exc:
39	35	if verbose:
40		print ~~exc~~
	36	print(exc)
41	37	else:
42	38	raise AssertionError("expected an exception")
43	39	try:
44		yaml.load(~~StringIO.String~~IO(input))
45		except yaml.YAMLError, exc:
	40	yaml.load(io.BytesIO(input))
	41	except yaml.YAMLError as exc:
46	42	if verbose:
47		print ~~exc~~
	43	print(exc)
48	44	else:
49	45	raise AssertionError("expected an exception")
…	…
54	50	data = open(unicode_filename, 'rb').read().decode('utf-8')
55	51	value = ' '.join(data.split())
56		for ~~encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le'~~]:
57		~~for allow_unicode in [False, True]:~~
58		~~data1 = yaml.dump(value, allow_unicode=allow_unicode)~~
59		stream = ~~StringIO~~.StringIO()
60		yaml.dump(value, ~~_unicode_open(stream, 'utf-8')~~, encoding=encoding, allow_unicode=allow_unicode)
	52	for allow_unicode in [False, True]:
	53	data1 = yaml.dump(value, allow_unicode=allow_unicode)
	54	for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']:
	55	stream = io.StringIO()
	56	yaml.dump(value, stream, encoding=encoding, allow_unicode=allow_unicode)
61	57	data2 = stream.getvalue()
62	58	data3 = yaml.dump(value, encoding=encoding, allow_unicode=allow_unicode)
63		stream = StringIO.StringIO()
64		yaml.dump(value, stream, encoding=encoding, allow_unicode=allow_unicode)
65		data4 = stream.getvalue()
	59	stream = io.BytesIO()
	60	if encoding is None:
	61	try:
	62	yaml.dump(value, stream, encoding=encoding, allow_unicode=allow_unicode)
	63	except TypeError as exc:
	64	if verbose:
	65	print(exc)
	66	data4 = None
	67	else:
	68	raise AssertionError("expected an exception")
	69	else:
	70	yaml.dump(value, stream, encoding=encoding, allow_unicode=allow_unicode)
	71	data4 = stream.getvalue()
	72	if verbose:
	73	print("BYTES:", data4[:50])
	74	data4 = data4.decode(encoding)
66	75	for copy in [data1, data2, data3, data4]:
	76	if copy is None:
	77	continue
	78	assert isinstance(copy, str)
67	79	if allow_unicode:
68	80	try:
69	81	copy[4:].encode('ascii')
70		except ~~(UnicodeDecodeError, UnicodeEncodeError),~~ exc:
	82	except UnicodeEncodeError as exc:
71	83	if verbose:
72		print ~~exc~~
	84	print(exc)
73	85	else:
74	86	raise AssertionError("expected an exception")
…	…
76	88	copy[4:].encode('ascii')
77	89	assert isinstance(data1, str), (type(data1), encoding)
78		~~data1.decode('utf-8')~~
79	90	assert isinstance(data2, str), (type(data2), encoding)
80		~~data2.decode('utf-8')~~
81		~~if encoding is None:~~
82		~~assert isinstance(data3, unicode), (type(data3), encoding)~~
83		~~assert isinstance(data4, unicode), (type(data4), encoding)~~
84		~~else:~~
85		~~assert isinstance(data3, str), (type(data3), encoding)~~
86		~~data3.decode(encoding)~~
87		~~assert isinstance(data4, str), (type(data4), encoding)~~
88		~~data4.decode(encoding)~~
89	91
90	92	test_unicode_output.unittest = ['.unicode']
…	…
95	97	input = data
96	98	if encoding is not None:
97		input = (u'\ufeff'+input).encode(encoding)
	99	input = ('\ufeff'+input).encode(encoding)
98	100	output1 = yaml.emit(yaml.parse(input), allow_unicode=True)
99		stream = StringIO.StringIO()
100		yaml.emit(yaml.parse(input), _unicode_open(stream, 'utf-8'),
101		allow_unicode=True)
	101	if encoding is None:
	102	stream = io.StringIO()
	103	else:
	104	stream = io.BytesIO()
	105	yaml.emit(yaml.parse(input), stream, allow_unicode=True)
102	106	output2 = stream.getvalue()
	107	assert isinstance(output1, str), (type(output1), encoding)
103	108	if encoding is None:
104		assert isinstance(output~~1, unicode~~), (type(output1), encoding)
	109	assert isinstance(output2, str), (type(output1), encoding)
105	110	else:
106		assert isinstance(output1, str), (type(output1), encoding)
107		output1.decode(encoding)
108		assert isinstance(output2, str), (type(output2), encoding)
109		output2.decode('utf-8')
	111	assert isinstance(output2, bytes), (type(output1), encoding)
	112	output2.decode(encoding)
110	113
111	114	test_unicode_transfer.unittest = ['.unicode']

pyyaml/trunk/tests/lib3/test_yaml.py

r330	r334
11	11	from test_representer import *
12	12	from test_recursive import *
	13	from test_input_output import *
13	14
14	15	if __name__ == '__main__':

pyyaml/trunk/tests/lib3/test_yaml_ext.py

r331	r334
262	262
263	263	import test_tokens, test_structure, test_errors, test_resolver, test_constructor, \
264		test_emitter, test_representer, test_recursive
	264	test_emitter, test_representer, test_recursive, test_input_output
265	265	wrap_ext([test_tokens, test_structure, test_errors, test_resolver, test_constructor,
266		test_emitter, test_representer, test_recursive])
	266	test_emitter, test_representer, test_recursive, test_input_output])
267	267
268	268	if __name__ == '__main__':

Download in other formats: