Changeset 136

Timestamp:

04/15/06 19:54:52 (6 years ago)

Author:

xi

Message:

Major refactoring.

Location:

pyyaml/trunk

Files:

• examples/yaml-hl/yaml_hl.py (modified) (1 diff)
• lib/yaml/__init__.py (modified) (2 diffs)
• lib/yaml/composer.py (modified) (4 diffs)
• lib/yaml/constructor.py (modified) (4 diffs)
• lib/yaml/detector.py (modified) (1 diff)
• lib/yaml/dumper.py (added)
• lib/yaml/emitter.py (modified) (34 diffs)
• lib/yaml/error.py (modified) (2 diffs)
• lib/yaml/events.py (modified) (2 diffs)
• lib/yaml/loader.py (added)
• lib/yaml/nodes.py (modified) (1 diff)
• lib/yaml/parser.py (modified) (29 diffs)
• lib/yaml/reader.py (modified) (2 diffs)
• lib/yaml/representer.py (modified) (5 diffs)
• lib/yaml/resolver.py (deleted)
• lib/yaml/scanner.py (modified) (63 diffs)
• lib/yaml/serializer.py (modified) (3 diffs)
• lib/yaml/yaml_object.py (deleted)
• tests/data/construct-custom.code (modified) (1 diff)
• tests/data/construct-custom.data (modified) (1 diff)
• tests/data/spec-08-07.canonical (modified) (1 diff)
• tests/data/tags.events (modified) (1 diff)
• tests/test_appliance.py (modified) (6 diffs)
• tests/test_canonical.py (modified) (2 diffs)
• tests/test_constructor.py (modified) (4 diffs)
• tests/test_detector.py (modified) (3 diffs)
• tests/test_emitter.py (modified) (6 diffs)
• tests/test_errors.py (modified) (2 diffs)
• tests/test_representer.py (modified) (4 diffs)
• tests/test_structure.py (modified) (8 diffs)
• tests/test_tokens.py (modified) (3 diffs)

pyyaml/trunk/examples/yaml-hl/yaml_hl.py

@@ -2 +2 @@
 
 import yaml, codecs, sys, optparse
+
+
+
+yaml.add_resolver(u'!Config', [])
+yaml.add_resolver(u'!TokensConfig', [u'tokens'])
+yaml.add_resolver(u'!EventsConfig', [u'events'])
+yaml.add_resolver(u'!StartEndConfig', [u'tokens', None])
+yaml.add_resolver(u'!StartEndConfig', [u'events', None])
 
 class YAMLHighlight:

pyyaml/trunk/lib/yaml/__init__.py

@@ -6 +6 @@
 from parser import *
 from composer import *
-from resolver import *
 from constructor import *
 
@@ -19 +18 @@
 from nodes import *
 
-from yaml_object import *
-
-def parse(data, Reader=Reader, Scanner=Scanner, Parser=Parser):
-    reader = Reader(data)
-    scanner = Scanner(reader)
-    parser = Parser(scanner)
-    return parser
-
-def load_all(data, Reader=Reader, Scanner=Scanner, Parser=Parser,
-        Composer=Composer, Resolver=Resolver, Constructor=Constructor):
-    reader = Reader(data)
-    scanner = Scanner(reader)
-    parser = Parser(scanner)
-    composer = Composer(parser)
-    resolver = Resolver(composer)
-    constructor = Constructor(resolver)
-    return constructor
-
-def safe_load_all(data, Reader=Reader, Scanner=Scanner, Parser=Parser,
-        Composer=Composer, Resolver=Resolver, Constructor=SafeConstructor):
-    return load_all(data, Reader, Scanner, Parser, Composer, Resolver,
-            Constructor)
-
-def load(data, *args, **kwds):
-    for document in load_all(data, *args, **kwds):
-        return document
-
-def safe_load(data, *args, **kwds):
-    for document in safe_load_all(data, *args, **kwds):
-        return document
-
-def emit(events, writer=None, Emitter=Emitter):
-    if writer is None:
+from loader import *
+from dumper import *
+
+def scan(stream, Loader=Loader):
+    """
+    Scan a YAML stream and produce scanning tokens.
+    """
+    loader = Loader(stream)
+    while loader.check_token():
+        yield loader.get_token()
+
+def parse(stream, Loader=Loader):
+    """
+    Parse a YAML stream and produce parsing events.
+    """
+    loader = Loader(stream)
+    while loader.check_event():
+        yield loader.get_event()
+
+def compose(stream, Loader=Loader):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding representation tree.
+    """
+    loader = Loader(stream)
+    if loader.check_node():
+        return loader.get_node()
+
+def compose_all(stream, Loader=Loader):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponsing representation trees.
+    """
+    loader = Loader(stream)
+    while loader.check_node():
+        yield loader.get_node()
+
+def load_all(stream, Loader=Loader):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+    """
+    loader = Loader(stream)
+    while loader.check_data():
+        yield loader.get_data()
+
+def load(stream, Loader=Loader):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+    """
+    loader = Loader(stream)
+    if loader.check_data():
+        return loader.get_data()
+
+def safe_load_all(stream):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+    Resolve only basic YAML tags.
+    """
+    return load_all(stream, SafeLoader)
+
+def safe_load(stream):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+    Resolve only basic YAML tags.
+    """
+    return load(stream, SafeLoader)
+
+def emit(events, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None):
+    """
+    Emit YAML parsing events into a stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
         try:
             from cStringIO import StringIO
         except ImportError:
             from StringIO import StringIO
-        writer = StringIO()
-        return_value = True
-    else:
-        return_value = False
-    emitter = Emitter(writer)
+        stream = StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break)
     for event in events:
-        emitter.emit(event)
-    if return_value:
-        return writer.getvalue()
-
-def dump_all(natives, writer=None, Emitter=Emitter,
-        Serializer=Serializer, Representer=Representer,
-        encoding='utf-8', line_break=None, canonical=None,
-        indent=None, width=None, allow_unicode=None):
-    if writer is None:
+        dumper.emit(event)
+    if getvalue:
+        return getvalue()
+
+def serialize_all(nodes, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None,
+        encoding='utf-8', explicit_start=None, explicit_end=None,
+        version=None, tags=None):
+    """
+    Serialize a sequence of representation trees into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
         try:
             from cStringIO import StringIO
         except ImportError:
             from StringIO import StringIO
-        writer = StringIO()
-        return_value = True
-    else:
-        return_value = False
-    emitter = Emitter(writer)
-    serializer = Serializer(emitter, encoding=encoding, line_break=line_break,
-            canonical=canonical, indent=indent, width=width,
-            allow_unicode=allow_unicode)
-    representer = Representer(serializer)
-    for native in natives:
-        representer.represent(native)
-    representer.close()
-    if return_value:
-        return writer.getvalue()
-
-def safe_dump_all(natives, writer=None, Emitter=Emitter,
-        Serializer=Serializer, Representer=SafeRepresenter,
-        encoding='utf-8', line_break=None, canonical=None,
-        indent=None, width=None, allow_unicode=None):
-    return dump_all(natives, writer, Emitter, Serializer, Representer,
-            encoding, line_break, canonical, indent, width, allow_unicode)
-
-def dump(native, *args, **kwds):
-    return dump_all([native], *args, **kwds)
-
-def safe_dump(native, *args, **kwds):
-    return safe_dump_all([native], *args, **kwds)
-
+        stream = StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break,
+            encoding=encoding, version=version, tags=tags,
+            explicit_start=explicit_start, explicit_end=explicit_end)
+    dumper.open()
+    for node in nodes:
+        dumper.serialize(node)
+    dumper.close()
+    if getvalue:
+        return getvalue()
+
+def serialize(node, stream=None, Dumper=Dumper, **kwds):
+    """
+    Serialize a representation tree into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    return serialize_all([node], stream, Dumper=Dumper, **kwds)
+
+def dump_all(documents, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None,
+        encoding='utf-8', explicit_start=None, explicit_end=None,
+        version=None, tags=None):
+    """
+    Serialize a sequence of Python objects into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        try:
+            from cStringIO import StringIO
+        except ImportError:
+            from StringIO import StringIO
+        stream = StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break,
+            encoding=encoding, version=version, tags=tags,
+            explicit_start=explicit_start, explicit_end=explicit_end)
+    dumper.open()
+    for data in documents:
+        dumper.represent(data)
+    dumper.close()
+    if getvalue:
+        return getvalue()
+
+def dump(data, stream=None, Dumper=Dumper, **kwds):
+    """
+    Serialize a Python object into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all([data], stream, Dumper=Dumper, **kwds)
+
+def safe_dump_all(documents, stream=None, **kwds):
+    """
+    Serialize a sequence of Python objects into a YAML stream.
+    Produce only basic YAML tags.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
+
+def safe_dump(data, stream=None, **kwds):
+    """
+    Serialize a Python object into a YAML stream.
+    Produce only basic YAML tags.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all([data], stream, Dumper=SafeDumper, **kwds)
+
+def add_detector(tag, regexp, first=None, Loader=Loader, Dumper=Dumper):
+    """
+    Add an implicit scalar detector.
+    If an implicit scalar value matches the given regexp,
+    the corresponding tag is assigned to the scalar.
+    first is a sequence of possible initial characters or None.
+    """
+    Loader.add_detector(tag, regexp, first)
+    Dumper.add_detector(tag, regexp, first)
+
+def add_resolver(tag, path, Loader=Loader):
+    """
+    Add a path based resolver for the given tag.
+    A path is a list of keys that forms a path
+    to a node in the representation tree.
+    Keys can be string values, integers, or None.
+    """
+    Loader.add_resolver(tag, path)
+
+def add_constructor(tag, constructor, Loader=Loader):
+    """
+    Add a constructor for the given tag.
+    Constructor is a function that accepts a Loader instance
+    and a node object and produces the corresponding Python object.
+    """
+    Loader.add_constructor(tag, constructor)
+
+def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader):
+    """
+    Add a multi-constructor for the given tag prefix.
+    Multi-constructor is called for a node if its tag starts with tag_prefix.
+    Multi-constructor accepts a Loader instance, a tag suffix,
+    and a node object and produces the corresponding Python object.
+    """
+    Loader.add_multi_constructor(tag_prefix, multi_constructor)
+
+class YAMLObjectMetaclass(type):
+    """
+    The metaclass for YAMLObject.
+    """
+    def __init__(cls, name, bases, kwds):
+        super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
+        if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
+            cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+            cls.yaml_dumper.add_representer(cls, cls.to_yaml)
+
+class YAMLObject(object):
+    """
+    An object that can dump itself to a YAML stream
+    and load itself from a YAML stream.
+    """
+
+    __metaclass__ = YAMLObjectMetaclass
+
+    yaml_loader = Loader
+    yaml_dumper = Dumper
+
+    yaml_tag = None
+    yaml_flow_style = None
+
+    def from_yaml(cls, loader, node):
+        """
+        Convert a representation node to a Python object.
+        """
+        return loader.construct_yaml_object(node, cls)
+    from_yaml = classmethod(from_yaml)
+
+    def to_yaml(cls, dumper, data):
+        """
+        Convert a Python object to a representation node.
+        """
+        return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
+                flow_style=cls.yaml_flow_style)
+    to_yaml = classmethod(to_yaml)
+

pyyaml/trunk/lib/yaml/composer.py

@@ -1 +1 @@
 
-__all__ = ['Composer', 'ComposerError']
+__all__ = ['BaseComposer', 'Composer', 'ComposerError']
 
 from error import MarkedYAMLError
@@ -9 +9 @@
     pass
 
-class Composer:
+class BaseComposer:
 
-    def __init__(self, parser):
-        self.parser = parser
+    yaml_resolvers = {}
+
+    def __init__(self):
         self.all_anchors = {}
         self.complete_anchors = {}
+        self.resolver_tags = []
+        self.resolver_paths = []
 
-        # Drop the STREAM-START event.
-        self.parser.get()
+    def check_node(self):
+        # If there are more documents available?
+        return not self.check_event(StreamEndEvent)
 
-    def check(self):
-        # If there are more documents available?
-        return not self.parser.check(StreamEndEvent)
-
-    def get(self):
+    def get_node(self):
         # Get the root node of the next document.
-        if not self.parser.check(StreamEndEvent):
+        if not self.check_event(StreamEndEvent):
             return self.compose_document()
 
     def __iter__(self):
         # Iterator protocol.
-        while not self.parser.check(StreamEndEvent):
+        while not self.check_event(StreamEndEvent):
             yield self.compose_document()
 
     def compose_document(self):
 
+        # Drop the STREAM-START event.
+        if self.check_event(StreamStartEvent):
+            self.get_event()
+
         # Drop the DOCUMENT-START event.
-        self.parser.get()
+        self.get_event()
 
         # Compose the root node.
-        node = self.compose_node()
+        node = self.compose_node([])
 
         # Drop the DOCUMENT-END event.
-        self.parser.get()
+        self.get_event()
 
         self.all_anchors = {}
         self.complete_anchors = {}
+        self.resolver_tags = []
+        self.resolver_paths = []
         return node
 
-    def compose_node(self):
-        if self.parser.check(AliasEvent):
-            event = self.parser.get()
+    def increase_resolver_depth(self, path):
+        depth = len(path)
+        tag = None
+        paths = []
+        if not depth:
+            for resolver_path in self.yaml_resolvers.keys():
+                if resolver_path:
+                    paths.append(resolver_path)
+                else:
+                    tag = self.yaml_resolvers[resolver_path]
+        else:
+            base, index = path[-1]
+            if isinstance(index, ScalarNode)    \
+                    and index.tag == self.DEFAULT_SCALAR_TAG:
+                index = index.value
+            elif isinstance(index, Node):
+                index = None
+            for resolver_path in self.resolver_paths[-1]:
+                resolver_index = resolver_path[depth-1]
+                if resolver_index is None or resolver_index == index:
+                    if len(resolver_index) > depth:
+                        paths.append(resolver_path)
+                    else:
+                        tag = self.yaml_resolvers[resolver_path]
+        self.resolver_tags.append(tag)
+        self.resolver_paths.append(paths)
+
+    def decrease_resolver_depth(self):
+        del self.resolver_tags[-1]
+        del self.resolver_paths[-1]
+
+    def compose_node(self, path):
+        if self.check_event(AliasEvent):
+            event = self.get_event()
             anchor = event.anchor
             if anchor not in self.all_anchors:
@@ -62 +99 @@
                         event.start_mark)
             return self.complete_anchors[anchor]
-        event = self.parser.peek()
+        self.increase_resolver_depth(path)
+        event = self.peek_event()
         anchor = event.anchor
         if anchor is not None:
@@ -70 +108 @@
                         "second occurence", event.start_mark)
             self.all_anchors[anchor] = event
-        if self.parser.check(ScalarEvent):
-            node = self.compose_scalar_node()
-        elif self.parser.check(SequenceStartEvent):
-            node = self.compose_sequence_node()
-        elif self.parser.check(MappingStartEvent):
-            node = self.compose_mapping_node()
+        if self.check_event(ScalarEvent):
+            node = self.compose_scalar_node(path)
+        elif self.check_event(SequenceStartEvent):
+            node = self.compose_sequence_node(path)
+        elif self.check_event(MappingStartEvent):
+            node = self.compose_mapping_node(path)
         if anchor is not None:
             self.complete_anchors[anchor] = node
+        self.decrease_resolver_depth()
         return node
 
-    def compose_scalar_node(self):
-        event = self.parser.get()
-        return ScalarNode(event.tag, event.value, event.implicit,
+    def compose_scalar_node(self, path):
+        event = self.get_event()
+        tag = self.resolve_scalar(path, event.tag, event.implicit, event.value)
+        return ScalarNode(tag, event.value,
                 event.start_mark, event.end_mark, style=event.style)
 
-    def compose_sequence_node(self):
-        start_event = self.parser.get()
-        value = []
-        while not self.parser.check(SequenceEndEvent):
-            value.append(self.compose_node())
-        end_event = self.parser.get()
-        return SequenceNode(start_event.tag, value,
-                start_event.start_mark, end_event.end_mark,
+    def compose_sequence_node(self, path):
+        start_event = self.get_event()
+        tag = self.resolve_sequence(path, start_event.tag)
+        node = SequenceNode(tag, [],
+                start_event.start_mark, None,
                 flow_style=start_event.flow_style)
+        index = 0
+        while not self.check_event(SequenceEndEvent):
+            node.value.append(self.compose_node(path+[(node, index)]))
+            index += 1
+        end_event = self.get_event()
+        node.end_mark = end_event.end_mark
+        return node
 
-    def compose_mapping_node(self):
-        start_event = self.parser.get()
-        value = {}
-        while not self.parser.check(MappingEndEvent):
-            key_event = self.parser.peek()
-            item_key = self.compose_node()
-            item_value = self.compose_node()
-            if item_key in value:
+    def compose_mapping_node(self, path):
+        start_event = self.get_event()
+        tag = self.resolve_mapping(path, start_event.tag)
+        node = MappingNode(tag, {},
+                start_event.start_mark, None,
+                flow_style=start_event.flow_style)
+        while not self.check_event(MappingEndEvent):
+            key_event = self.peek_event()
+            item_key = self.compose_node(path+[(node, None)])
+            item_value = self.compose_node(path+[(node, item_key)])
+            if item_key in node.value:
                 raise ComposerError("while composing a mapping", start_event.start_mark,
                         "found duplicate key", key_event.start_mark)
-            value[item_key] = item_value
-        end_event = self.parser.get()
-        return MappingNode(start_event.tag, value,
-                start_event.start_mark, end_event.end_mark,
-                flow_style=start_event.flow_style)
+            node.value[item_key] = item_value
+        end_event = self.get_event()
+        node.end_mark = end_event.end_mark
+        return node
 
+    def resolve_scalar(self, path, tag, implicit, value):
+        if implicit:
+            tag = self.detect(value)
+        if tag is None and self.resolver_tags[-1]:
+            tag = self.resolver_tags[-1]
+        if tag is None or tag == u'!':
+            tag = self.DEFAULT_SCALAR_TAG
+        return tag
+
+    def resolve_sequence(self, path, tag):
+        if tag is None and self.resolver_tags[-1]:
+            tag = self.resolver_tags[-1]
+        if tag is None or tag == u'!':
+            tag = self.DEFAULT_SEQUENCE_TAG
+        return tag
+
+    def resolve_mapping(self, path, tag):
+        if tag is None and self.resolver_tags[-1]:
+            tag = self.resolver_tags[-1]
+        if tag is None or tag == u'!':
+            tag = self.DEFAULT_MAPPING_TAG
+        return tag
+
+    def add_resolver(self, tag, path):
+        if not 'yaml_resolvers' in cls.__dict__:
+            cls.yaml_resolvers = cls.yaml_resolvers.copy()
+        cls.yaml_resolvers[tuple(path)] = tag
+    add_resolver = classmethod(add_resolver)
+
+class Composer(BaseComposer):
+    pass
+

pyyaml/trunk/lib/yaml/constructor.py

@@ -5 +5 @@
 from error import *
 from nodes import *
+from composer import *
 
 try:
@@ -22 +23 @@
     pass
 
-class BaseConstructor:
-
-    def __init__(self, resolver):
-        self.resolver = resolver
+class BaseConstructor(Composer):
+
+    yaml_constructors = {}
+    yaml_multi_constructors = {}
+
+    def __init__(self):
         self.constructed_objects = {}
 
-    def check(self):
+    def check_data(self):
         # If there are more documents available?
-        return self.resolver.check()
-
-    def get(self):
+        return self.check_node()
+
+    def get_data(self):
         # Construct and return the next document.
-        if self.resolver.check():
-            return self.construct_document(self.resolver.get())
+        if self.check_node():
+            return self.construct_document(self.get_node())
 
     def __iter__(self):
         # Iterator protocol.
-        while self.resolver.check():
-            yield self.construct_document(self.resolver.get())
+        while self.check_node():
+            yield self.construct_document(self.get_node())
 
     def construct_document(self, node):
-        native = self.construct_object(node)
+        data = self.construct_object(node)
         self.constructed_objects = {}
-        return native
+        return data
 
     def construct_object(self, node):
         if node in self.constructed_objects:
             return self.constructed_objects[node]
+        constructor = None
         if node.tag in self.yaml_constructors:
-            native = self.yaml_constructors[node.tag](self, node)
-        elif None in self.yaml_constructors:
-            native = self.yaml_constructors[None](self, node)
-        elif isinstance(node, ScalarNode):
-            native = self.construct_scalar(node)
-        elif isinstance(node, SequenceNode):
-            native = self.construct_sequence(node)
-        elif isinstance(node, MappingNode):
-            native = self.construct_mapping(node)
-        self.constructed_objects[node] = native
-        return native
+            constructor = lambda node: self.yaml_constructors[node.tag](self, node)
+        else:
+            for tag_prefix in self.yaml_multi_constructors:
+                if node.tag.startswith(tag_prefix):
+                    tag_suffix = node.tag[len(tag_prefix):]
+                    constructor = lambda node:  \
+                            self.yaml_multi_constructors[tag_prefix](self, tag_suffix, node)
+                break
+            else:
+                if None in self.yaml_multi_constructors:
+                    constructor = lambda node:  \
+                            self.yaml_multi_constructors[None](self, node.tag, node)
+                elif None in self.yaml_constructors:
+                    constructor = lambda node:  \
+                            self.yaml_constructors[None](self, node)
+                elif isinstance(node, ScalarNode):
+                    constructor = self.construct_scalar
+                elif isinstance(node, SequenceNode):
+                    constructor = self.construct_sequence
+                elif isinstance(node, MappingNode):
+                    constructor = self.construct_mapping
+        data = constructor(node)
+        self.constructed_objects[node] = data
+        return data
 
     def construct_scalar(self, node):
@@ -153 +170 @@
     add_constructor = classmethod(add_constructor)
 
-    yaml_constructors = {}
+    def add_multi_constructor(cls, tag_prefix, multi_constructor):
+        if not 'yaml_multi_constructors' in cls.__dict__:
+            cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
+        cls.yaml_multi_constructors[tag_prefix] = multi_constructor
+    add_multi_constructor = classmethod(add_multi_constructor)
 
 class SafeConstructor(BaseConstructor):
@@ -328 +349 @@
         return self.construct_mapping(node)
 
+    def construct_yaml_object(self, node, cls):
+        mapping = self.construct_mapping(node)
+        state = {}
+        for key in mapping:
+            state[key.replace('-', '_')] = mapping[key]
+        data = cls.__new__(cls)
+        if hasattr(data, '__setstate__'):
+            data.__setstate__(mapping)
+        else:
+            data.__dict__.update(mapping)
+        return data
+
     def construct_undefined(self, node):
         raise ConstructorError(None, None,

pyyaml/trunk/lib/yaml/detector.py

@@ -6 +6 @@
 class BaseDetector:
 
+    DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str'
+    DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq'
+    DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map'
+
     yaml_detectors = {}
+
+    def __init__(self):
+        pass
 
     def add_detector(cls, tag, regexp, first):

pyyaml/trunk/lib/yaml/emitter.py

@@ -12 +12 @@
 from events import *
 
+import re
+
 class EmitterError(YAMLError):
     pass
@@ -18 +20 @@
     def __init__(self, scalar, empty, multiline,
             allow_flow_plain, allow_block_plain,
-            allow_single_quoted, allow_double_quoted, allow_block):
+            allow_single_quoted, allow_double_quoted,
+            allow_block):
         self.scalar = scalar
         self.empty = empty
@@ -35 +38 @@
     }
 
-    def __init__(self, writer):
-
-        # The writer should have the methods `write` and possibly `flush`.
-        self.writer = writer
-
-        # Encoding is provided by STREAM-START.
+    def __init__(self, stream, canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None):
+
+        # The stream should have the methods `write` and possibly `flush`.
+        self.stream = stream
+
+        # Encoding can be overriden by STREAM-START.
         self.encoding = None
 
@@ -76 +80 @@
 
         # Formatting details.
-        self.canonical = False
-        self.allow_unicode = False
+        self.canonical = canonical
+        self.allow_unicode = allow_unicode
+        self.best_indent = 2
+        if indent and 1 < indent < 10:
+            self.best_indent = indent
+        self.best_width = 80
+        if width and width > self.best_indent*2:
+            self.best_width = width
         self.best_line_break = u'\n'
-        self.best_indent = 2
-        self.best_width = 80
+        if line_break in [u'\r', u'\n', u'\r\n']:
+            self.best_line_break = line_break
+
+        # Tag prefixes.
         self.tag_prefixes = None
 
-        # Analyses cache.
-        self.anchor_text = None
-        self.tag_text = None
-        self.scalar_analysis = None
-        self.scalar_style = None
+        # Prepared anchor and tag.
+        self.prepared_anchor = None
+        self.prepared_tag = None
+
+        # Scalar analysis and style.
+        self.analysis = None
+        self.style = None
 
     def emit(self, event):
@@ -140 +154 @@
     def expect_stream_start(self):
         if isinstance(self.event, StreamStartEvent):
-            self.encoding = self.event.encoding
-            self.canonical = self.event.canonical
-            self.allow_unicode = self.event.allow_unicode
-            if self.event.indent and self.event.indent > 1:
-                self.best_indent = self.event.indent
-            if self.event.width and self.event.width > self.best_indent:
-                self.best_width = self.event.width
-            if self.event.line_break in [u'\r', u'\n', u'\r\n']:
-                self.best_line_break = self.event.line_break
+            if self.event.encoding:
+                self.encoding = self.event.encoding
             self.write_stream_start()
             self.state = self.expect_first_document_start
@@ -166 +173 @@
         if isinstance(self.event, DocumentStartEvent):
             if self.event.version:
-                version_text = self.analyze_version(self.event.version)
+                version_text = self.prepare_version(self.event.version)
                 self.write_version_directive(version_text)
             self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
@@ -175 +182 @@
                     prefix = self.event.tags[handle]
                     self.tag_prefixes[prefix] = handle
-                    handle_text = self.analyze_tag_handle(handle)
-                    prefix_text = self.analyze_tag_prefix(prefix)
+                    handle_text = self.prepare_tag_handle(handle)
+                    prefix_text = self.prepare_tag_prefix(prefix)
                     self.write_tag_directive(handle_text, prefix_text)
             implicit = (first and not self.event.explicit and not self.canonical
@@ -200 +207 @@
                 self.write_indicator(u'...', True)
                 self.write_indent()
+            self.flush_stream()
             self.state = self.expect_document_start
         else:
@@ -419 +427 @@
         length = 0
         if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
-            if self.anchor_text is None:
-                self.anchor_text = self.analyze_anchor(self.event.anchor)
-            length += len(self.anchor_text)
+            if self.prepared_anchor is None:
+                self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+            length += len(self.prepared_anchor)
         if isinstance(self.event, (ScalarEvent, CollectionStartEvent))  \
                 and self.event.tag is not None:
-            if self.tag_text is None:
-                self.tag_text = self.analyze_tag(self.event.tag)
-            length += len(self.tag_text)
+            if self.prepared_tag is None:
+                self.prepared_tag = self.prepare_tag(self.event.tag)
+            length += len(self.prepared_tag)
         if isinstance(self.event, ScalarEvent):
-            if self.scalar_analysis is None:
-                self.scalar_analysis = self.analyze_scalar(self.event.value)
-            length += len(self.scalar_analysis.scalar)
+            if self.analysis is None:
+                self.analysis = self.analyze_scalar(self.event.value)
+            length += len(self.analysis.scalar)
         return (length < 128 and (isinstance(self.event, AliasEvent)
-            or (isinstance(self.event, ScalarEvent) and not self.scalar_analysis.multiline)
+            or (isinstance(self.event, ScalarEvent)
+                    and not self.analysis.empty and not self.analysis.multiline)
             or self.check_empty_sequence() or self.check_empty_mapping()))
 
@@ -439 +448 @@
     def process_anchor(self, indicator):
         if self.event.anchor is None:
+            self.prepared_anchor = None
             return
-        if self.anchor_text is None:
-            self.anchor_text = self.analyze_anchor(self.event.anchor)
-        if self.anchor_text:
-            self.write_indicator(indicator+self.anchor_text, True)
-        self.anchor_text = None
+        if self.prepared_anchor is None:
+            self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+        if self.prepared_anchor:
+            self.write_indicator(indicator+self.prepared_anchor, True)
+        self.prepared_anchor = None
 
     def process_tag(self):
-        if self.event.tag is None:
+        tag = self.event.tag
+        if isinstance(self.event, ScalarEvent):
+            if self.style is None:
+                self.style = self.choose_scalar_style()
+            if self.style == '':
+                self.prepared_tag = None
+                return
+            if self.event.implicit and not tag:
+                tag = u'!'
+                self.prepared_tag = None
+        if not tag:
+            self.prepared_tag = None
             return
-        if isinstance(self.event, ScalarEvent) and self.best_scalar_style() == '':
-            return
-        if self.tag_text is None:
-            self.tag_text = self.analyze_tag(self.event.tag)
-        if self.tag_text:
-            self.write_indicator(self.tag_text, True)
-        self.tag_text = None
-
-    def best_scalar_style(self):
-        if self.scalar_analysis is None:
-            self.scalar_analysis = self.analyze_scalar(self.event.value)
-        if self.canonical:
+        if self.prepared_tag is None:
+            self.prepared_tag = self.prepare_tag(tag)
+        if self.prepared_tag:
+            self.write_indicator(self.prepared_tag, True)
+        self.prepared_tag = None
+
+    def choose_scalar_style(self):
+        if self.analysis is None:
+            self.analysis = self.analyze_scalar(self.event.value)
+        if self.event.style == '"' or self.canonical:
             return '"'
-        if (self.event.implicit and not self.event.style
-                and ((self.flow_level and self.scalar_analysis.allow_flow_plain)
-                    or (not self.flow_level and self.scalar_analysis.allow_block_plain))
-                and (len(self.scalar_analysis.scalar) > 0
-                    or (not self.flow_level and not self.simple_key_context))):
-            return ''
-        elif self.event.style == '\'' and self.scalar_analysis.allow_single_quoted:
-            return '\''
-        elif self.event.style in ['|', '>'] and not self.flow_level and self.scalar_analysis.allow_block:
-            return self.event.style
-        else:
-            return '"'
-        return style
+        if not self.event.style and self.event.implicit:
+            if (not (self.simple_key_context and
+                    (self.analysis.empty or self.analysis.multiline))
+                and (self.flow_level and self.analysis.allow_flow_plain
+                    or (not self.flow_level and self.analysis.allow_block_plain))):
+                return ''
+        if self.event.style and self.event.style in '|>':
+            if not self.flow_level and self.analysis.allow_block:
+                return self.event.style
+        if not self.event.style or self.event.style == '\'':
+            if (self.analysis.allow_single_quoted and
+                    not (self.simple_key_context and self.analysis.multiline)):
+                return '\''
+        return '"'
 
     def process_scalar(self):
-        if self.scalar_analysis is None:
-            self.scalar_analysis = self.analyze_scalar(self.event.value)
-        style = self.best_scalar_style()
-        if self.scalar_analysis.multiline and not self.simple_key_context   \
-                and style not in ['|', '>']:
-            self.write_indent()
-        if style == '"':
-            self.write_double_quoted(self.scalar_analysis.scalar,
-                    split=(not self.simple_key_context))
-        elif style == '\'':
-            self.write_single_quoted(self.scalar_analysis.scalar,
-                    split=(not self.simple_key_context))
-        elif style == '>':
-            self.write_folded(self.scalar_analysis.scalar)
-        elif style == '|':
-            self.write_literal(self.scalar_analysis.scalar)
-        else:
-            self.write_plain(self.scalar_analysis.scalar,
-                    split=(not self.simple_key_context))
-        self.scalar_analysis = None
+        if self.analysis is None:
+            self.analysis = self.analyze_scalar(self.event.value)
+        if self.style is None:
+            self.style = self.choose_scalar_style()
+        split = (not self.simple_key_context)
+        #if self.analysis.multiline and split    \
+        #        and (not self.style or self.style in '\'\"'):
+        #    self.write_indent()
+        if self.style == '"':
+            self.write_double_quoted(self.analysis.scalar, split)
+        elif self.style == '\'':
+            self.write_single_quoted(self.analysis.scalar, split)
+        elif self.style == '>':
+            self.write_folded(self.analysis.scalar)
+        elif self.style == '|':
+            self.write_literal(self.analysis.scalar)
+        else:
+            self.write_plain(self.analysis.scalar, split)
+        self.analysis = None
+        self.style = None
 
     # Analyzers.
 
-    def analyze_version(self, version):
+    def prepare_version(self, version):
         major, minor = version
         if major != 1:
@@ -506 +526 @@
         return u'%d.%d' % (major, minor)
 
-    def analyze_tag_handle(self, handle):
+    def prepare_tag_handle(self, handle):
         if not handle:
             raise EmitterError("tag handle must not be empty")
@@ -519 +539 @@
         return handle
 
-    def analyze_tag_prefix(self, prefix):
+    def prepare_tag_prefix(self, prefix):
         if not prefix:
             raise EmitterError("tag prefix must not be empty")
@@ -542 +562 @@
         return u''.join(chunks)
 
-    def analyze_tag(self, tag):
+    def prepare_tag(self, tag):
         if not tag:
             raise EmitterError("tag must not be empty")
+        if tag == u'!':
+            return tag
         handle = None
         suffix = tag
@@ -575 +597 @@
             return u'!<%s>' % suffix_text
 
-    def analyze_anchor(self, anchor):
+    def prepare_anchor(self, anchor):
         if not anchor:
             raise EmitterError("anchor must not be empty")
@@ -585 +607 @@
         return anchor
 
-    def analyze_scalar(self, scalar):   # It begs for refactoring.
+    def analyze_scalar(self, scalar):
+
+        # Empty scalar is a special case.
         if not scalar:
             return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
@@ -591 +615 @@
                     allow_single_quoted=True, allow_double_quoted=True,
                     allow_block=False)
-        contains_block_indicator = False
-        contains_flow_indicator = False
-        contains_line_breaks = False
-        contains_unicode_characters = False
-        contains_special_characters = False
-        contains_inline_spaces = False          # non-space space+ non-space
-        contains_inline_breaks = False          # non-space break+ non-space
-        contains_leading_spaces = False         # ^ space+ (non-space | $)
-        contains_leading_breaks = False         # ^ break+ (non-space | $)
-        contains_trailing_spaces = False        # non-space space+ $
-        contains_trailing_breaks = False        # non-space break+ $
-        contains_inline_breaks_spaces = False   # non-space break+ space+ non-space
-        contains_mixed_breaks_spaces = False    # anything else
+
+        # Indicators and special characters.
+        block_indicators = False
+        flow_indicators = False
+        line_breaks = False
+        special_characters = False
+
+        # Whitespaces.
+        inline_spaces = False          # non-space space+ non-space
+        inline_breaks = False          # non-space break+ non-space
+        leading_spaces = False         # ^ space+ (non-space | $)
+        leading_breaks = False         # ^ break+ (non-space | $)
+        trailing_spaces = False        # (^ | non-space) space+ $
+        trailing_breaks = False        # (^ | non-space) break+ $
+        inline_breaks_spaces = False   # non-space break+ space+ non-space
+        mixed_breaks_spaces = False    # anything else
+
+        # Check document indicators.
         if scalar.startswith(u'---') or scalar.startswith(u'...'):
-            contains_block_indicator = True
-            contains_flow_indicator = True
-        first = True
-        last = (len(scalar) == 1)
-        preceeded_by_space = False
-        followed_by_space = (len(scalar) > 1 and
+            block_indicators = True
+            flow_indicators = True
+
+        # First character or preceded by a whitespace.
+        preceeded_by_space = True
+
+        # Last character or followed by a whitespace.
+        followed_by_space = (len(scalar) == 1 or
                 scalar[1] in u'\0 \t\r\n\x85\u2028\u2029')
-        spaces = breaks = mixed = leading = False
+
+        # The current series of whitespaces contain plain spaces.
+        spaces = False
+
+        # The current series of whitespaces contain line breaks.
+        breaks = False
+
+        # The current series of whitespaces contain a space followed by a
+        # break.
+        mixed = False
+
+        # The current series of whitespaces start at the beginning of the
+        # scalar.
+        leading = False
+
         index = 0
         while index < len(scalar):
             ch = scalar[index]
-            if first:
+
+            # Check for indicators.
+
+            if index == 0:
+                # Leading indicators are special characters.
                 if ch in u'#,[]{}#&*!|>\'\"%@`': 
-                    contains_flow_indicator = True
-                    contains_block_indicator = True
+                    flow_indicators = True
+                    block_indicators = True
                 if ch in u'?:':
-                    contains_flow_indicator = True
-                    if followed_by_space or last:
-                        contains_block_indicator = True
-                if ch == u'-' and (followed_by_space or last):
-                    contains_flow_indicator = True
-                    contains_block_indicator = True
+                    flow_indicators = True
+                    if followed_by_space:
+                        block_indicators = True
+                if ch == u'-' and followed_by_space:
+                    flow_indicators = True
+                    block_indicators = True
             else:
+                # Some indicators cannot appear within a scalar as well.
                 if ch in u',?[]{}':
-                    contains_flow_indicator = True
+                    flow_indicators = True
                 if ch == u':':
-                    contains_flow_indicator = True
-                    if followed_by_space or last:
-                        contains_block_indicator = True
-                if ch == u'#' and (preceeded_by_space or first):
-                    contains_flow_indicator = True
-                    contains_block_indicator = True
+                    flow_indicators = True
+                    if followed_by_space:
+                        block_indicators = True
+                if ch == u'#' and preceeded_by_space:
+                    flow_indicators = True
+                    block_indicators = True
+
+            # Check for line breaks, special, and unicode characters.
+
             if ch in u'\n\x85\u2028\u2029':
-                contains_line_breaks = True
+                line_breaks = True
             if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'):
-                if ch < u'\x80':
-                    contains_special_characters = True
+                if ch < u'\x80' or ch == u'\uFEFF': # '\uFEFF' is BOM.
+                    special_characters = True
                 else:
-                    contains_unicode_characters = True
-            if ch == u' ':
-                if not spaces and not breaks:
-                    leading = first
-                spaces = True
-            elif ch in u'\n\x85\u2028\u2029':
-                if not spaces and not breaks:
-                    leading = first
-                breaks = True
-                if spaces:
-                    mixed = True
-            if ch not in u' \n\x85\u2028\u2029':
+                    unicode_characters = True
+                    if not self.allow_unicode:
+                        special_characters = True
+
+            # Spaces, line breaks, and how they are mixed. State machine.
+
+            # Start or continue series of whitespaces.
+            if ch in u' \n\x85\u2028\u2029':
+                if spaces and breaks:
+                    if ch != u' ':      # break+ (space+ break+)    => mixed
+                        mixed = True
+                elif spaces:
+                    if ch != u' ':      # (space+ break+)   => mixed
+                        breaks = True
+                        mixed = True
+                elif breaks:
+                    if ch == u' ':      # break+ space+
+                        spaces = True
+                else:
+                    leading = (index == 0)
+                    if ch == u' ':      # space+
+                        spaces = True
+                    else:               # break+
+                        breaks = True
+
+            # Series of whitespaces ended with a non-space.
+            elif spaces or breaks:
                 if leading:
                     if spaces and breaks:
-                        contains_mixed_breaks_spaces = True
+                        mixed_breaks_spaces = True
                     elif spaces:
-                        contains_leading_spaces = True
+                        leading_spaces = True
                     elif breaks:
-                        contains_leading_breaks = True
+                        leading_breaks = True
                 else:
                     if mixed:
-                        contains_mixed_break_spaces = True
+                        mixed_breaks_spaces = True
                     elif spaces and breaks:
-                        contains_inline_breaks_spaces = True
+                        inline_breaks_spaces = True
                     elif spaces:
-                        contains_inline_spaces = True
+                        inline_spaces = True
                     elif breaks:
-                        contains_inline_breaks = True
+                        inline_breaks = True
                 spaces = breaks = mixed = leading = False
-            elif last:
+
+            # Series of whitespaces reach the end.
+            if (spaces or breaks) and (index == len(scalar)-1):
                 if spaces and breaks:
-                    contains_mixed_break_spaces = True
+                    mixed_breaks_spaces = True
                 elif spaces:
+                    trailing_spaces = True
                     if leading:
-                        contains_leading_spaces = True
-                    else:
-                        contains_trailing_spaces = True
+                        leading_spaces = True
                 elif breaks:
+                    trailing_breaks = True
                     if leading:
-                        contains_leading_breaks = True
-                    else:
-                        contains_trailing_breaks = True
+                        leading_breaks = True
+                spaces = breaks = mixed = leading = False
+
+            # Prepare for the next character.
             index += 1
-            first = False
-            last = (index+1 == len(scalar))
             preceeded_by_space = (ch in u'\0 \t\r\n\x85\u2028\u2029')
-            followed_by_space = (index+1 < len(scalar) and
+            followed_by_space = (index+1 >= len(scalar) or
                     scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029')
-        if contains_unicode_characters and not self.allow_unicode:
-            contains_special_characters = True
-        allow_flow_plain = not (contains_flow_indicator or contains_special_characters
-            or contains_leading_spaces or contains_leading_breaks
-            or contains_trailing_spaces or contains_trailing_breaks
-            or contains_inline_breaks_spaces or contains_mixed_breaks_spaces)
-        allow_block_plain = not (contains_block_indicator or contains_special_characters
-            or contains_leading_spaces or contains_leading_breaks
-            or contains_trailing_spaces or contains_trailing_breaks
-            or contains_inline_breaks_spaces or contains_mixed_breaks_spaces)
-        allow_single_quoted = not (contains_special_characters
-            or contains_inline_breaks_spaces or contains_mixed_breaks_spaces)
+
+        # Let's decide what styles are allowed.
+        allow_flow_plain = True
+        allow_block_plain = True
+        allow_single_quoted = True
         allow_double_quoted = True
-        allow_block = not (contains_special_characters
-            or contains_leading_spaces or contains_leading_breaks
-            or contains_trailing_spaces or contains_mixed_breaks_spaces)
-        return ScalarAnalysis(scalar=scalar, empty=False, multiline=contains_line_breaks,
-                allow_flow_plain=allow_flow_plain, allow_block_plain=allow_block_plain,
-                allow_single_quoted=allow_single_quoted, allow_double_quoted=allow_double_quoted,
+        allow_block = True
+
+        # Leading and trailing whitespace are bad for plain scalars. We also
+        # do not want to mess with leading whitespaces for block scalars.
+        if leading_spaces or leading_breaks or trailing_spaces:
+            allow_flow_plain = allow_block_plain = allow_block = False
+
+        # Trailing breaks are fine for block scalars, but unacceptable for
+        # plain scalars.
+        if trailing_breaks:
+            allow_flow_plain = allow_block_plain = False
+
+        # The combination of (space+ break+) is only acceptable for block
+        # scalars.
+        if inline_breaks_spaces:
+            allow_flow_plain = allow_block_plain = allow_single_quoted = False
+
+        # Mixed spaces and breaks, as well as special character are only
+        # allowed for double quoted scalars.
+        if mixed_breaks_spaces or special_characters:
+            allow_flow_plain = allow_block_plain =  \
+            allow_single_quoted = allow_block = False
+
+        # We don't emit multiline plain scalars.
+        if line_breaks:
+            allow_flow_plain = allow_block_plain = False
+
+        # Flow indicators are forbidden for flow plain scalars.
+        if flow_indicators:
+            allow_flow_plain = False
+
+        # Block indicators are forbidden for block plain scalars.
+        if block_indicators:
+            allow_block_plain = False
+
+        return ScalarAnalysis(scalar=scalar,
+                empty=False, multiline=line_breaks,
+                allow_flow_plain=allow_flow_plain,
+                allow_block_plain=allow_block_plain,
+                allow_single_quoted=allow_single_quoted,
+                allow_double_quoted=allow_double_quoted,
                 allow_block=allow_block)
 
     # Writers.
+
+    def flush_stream(self):
+        if hasattr(self.stream, 'flush'):
+            self.stream.flush()
 
     def write_stream_start(self):
         # Write BOM if needed.
         if self.encoding and self.encoding.startswith('utf-16'):
-            self.writer.write(u'\xFF\xFE'.encode(self.encoding))
+            self.stream.write(u'\xFF\xFE'.encode(self.encoding))
 
     def write_stream_end(self):
-        if hasattr(self.writer, 'flush'):
-            self.writer.flush()
+        self.flush_stream()
 
     def write_indicator(self, indicator, need_whitespace,
@@ -734 +834 @@
         if self.encoding:
             data = data.encode(self.encoding)
-        self.writer.write(data)
+        self.stream.write(data)
 
     def write_indent(self):
@@ -747 +847 @@
             if self.encoding:
                 data = data.encode(self.encoding)
-            self.writer.write(data)
+            self.stream.write(data)
 
     def write_line_break(self, data=None):
@@ -758 +858 @@
         if self.encoding:
             data = data.encode(self.encoding)
-        self.writer.write(data)
+        self.stream.write(data)
 
     def write_version_directive(self, version_text):
@@ -764 +864 @@
         if self.encoding:
             data = data.encode(self.encoding)
-        self.writer.write(data)
+        self.stream.write(data)
         self.write_line_break()
 
@@ -771 +871 @@
         if self.encoding:
             data = data.encode(self.encoding)
-        self.writer.write(data)
+        self.stream.write(data)
         self.write_line_break()
 
-    # Scalar writers.
+    # Scalar streams.
 
     def write_single_quoted(self, text, split=True):
@@ -795 +895 @@
                         if self.encoding:
                             data = data.encode(self.encoding)
-                        self.writer.write(data)
+                        self.stream.write(data)
                     start = end
             elif breaks:
@@ -815 +915 @@
                         if self.encoding:
                             data = data.encode(self.encoding)
-                        self.writer.write(data)
+                        self.stream.write(data)
                         start = end
                     if ch == u'\'':
@@ -822 +922 @@
                         if self.encoding:
                             data = data.encode(self.encoding)
-                        self.writer.write(data)
+                        self.stream.write(data)
                         start = end + 1
             if ch is not None:
@@ -864 +964 @@
                     if self.encoding:
                         data = data.encode(self.encoding)
-                    self.writer.write(data)
+                    self.stream.write(data)
                     start = end
                 if ch is not None:
@@ -878 +978 @@
                     if self.encoding:
                         data = data.encode(self.encoding)
-                    self.writer.write(data)
+                    self.stream.write(data)
                     start = end+1
             if 0 < end < len(text)-1 and (ch == u' ' or start >= end)   \
@@ -888 +988 @@
                 if self.encoding:
                     data = data.encode(self.encoding)
-                self.writer.write(data)
+                self.stream.write(data)
                 self.write_indent()
                 self.whitespace = False
@@ -897 +997 @@
                     if self.encoding:
                         data = data.encode(self.encoding)
-                    self.writer.write(data)
+                    self.stream.write(data)
             end += 1
         self.write_indicator(u'"', False)
@@ -948 +1048 @@
                         if self.encoding:
                             data = data.encode(self.encoding)
-                        self.writer.write(data)
+                        self.stream.write(data)
                     start = end
             else:
@@ -955 +1055 @@
                     if self.encoding:
                         data = data.encode(self.encoding)
-                    self.writer.write(data)
+                    self.stream.write(data)
                     if ch is None:
                         self.write_line_break()
@@ -989 +1089 @@
                     if self.encoding:
                         data = data.encode(self.encoding)
-                    self.writer.write(data)
+                    self.stream.write(data)
                     if ch is None:
                         self.write_line_break()
@@ -1005 +1105 @@
             if self.encoding:
                 data = data.encode(self.encoding)
-            self.writer.write(data)
+            self.stream.write(data)
         self.writespace = False
         self.indention = False
@@ -1026 +1126 @@
                         if self.encoding:
                             data = data.encode(self.encoding)
-                        self.writer.write(data)
+                        self.stream.write(data)
                     start = end
             elif breaks:
@@ -1047 +1147 @@
                     if self.encoding:
                         data = data.encode(self.encoding)
-                    self.writer.write(data)
+                    self.stream.write(data)
                     start = end
             if ch is not None:

pyyaml/trunk/lib/yaml/error.py

@@ -49 +49 @@
 
     def __init__(self, context=None, context_mark=None,
-            problem=None, problem_mark=None):
+            problem=None, problem_mark=None, note=None):
         self.context = context
         self.context_mark = context_mark
         self.problem = problem
         self.problem_mark = problem_mark
+        self.note = note
 
     def __str__(self):
         lines = []
-        #for (place, mark) in [(self.context, self.context_mark),
-        #                        (self.problem, self.problem_mark)]:
-        #    if place is not None:
-        #        lines.append(place)
-        #        if mark is not None:
-        #            lines.append(str(mark))
         if self.context is not None:
             lines.append(self.context)
@@ -75 +70 @@
         if self.problem_mark is not None:
             lines.append(str(self.problem_mark))
+        if self.note is not None:
+            lines.append(self.note)
         return '\n'.join(lines)
 
-
-

pyyaml/trunk/lib/yaml/events.py

@@ -34 +34 @@
 
 class StreamStartEvent(Event):
-    def __init__(self, start_mark=None, end_mark=None,
-            encoding=None, line_break=None, canonical=None,
-            indent=None, width=None, allow_unicode=None):
+    def __init__(self, start_mark=None, end_mark=None, encoding=None):
         self.start_mark = start_mark
         self.end_mark = end_mark
         self.encoding = encoding
-        self.line_break = line_break
-        self.canonical = canonical
-        self.indent = indent
-        self.width = width
-        self.allow_unicode = allow_unicode
 
 class StreamEndEvent(Event):
@@ -69 +62 @@
 
 class ScalarEvent(NodeEvent):
-    def __init__(self, anchor, tag, value, start_mark=None, end_mark=None,
-            implicit=None, style=None):
+    def __init__(self, anchor, tag, implicit, value,
+            start_mark=None, end_mark=None, style=None):
         self.anchor = anchor
         self.tag = tag
+        self.implicit = implicit
         self.value = value
         self.start_mark = start_mark
         self.end_mark = end_mark
-        self.implicit = implicit
         self.style = style
 

pyyaml/trunk/lib/yaml/nodes.py

@@ -24 +24 @@
 class ScalarNode(Node):
     id = 'scalar'
-    def __init__(self, tag, value, implicit,
+    def __init__(self, tag, value,
             start_mark=None, end_mark=None, style=None):
         self.tag = tag
         self.value = value
-        self.implicit = implicit
         self.start_mark = start_mark
         self.end_mark = end_mark

pyyaml/trunk/lib/yaml/parser.py

@@ -25 +25 @@
 # stream ::= (BOM? implicit_document)? (BOM? explicit_document)* STREAM-END
 
-# Note that there is a slight deviation from the specification. We require a
-# non-empty node content if ANCHOR or TAG is specified. This disallow such
-# documents as
-#
-#   key:    !!str   # empty value
-#
-# This is done to prevent ambiguity in parsing tags and aliases:
-#
-#   {   !!perl/YAML::Parser:    value }
-#
-# What is it? Should it be interpreted as
-#   {   ? !<tag:yaml.org,2002:perl/YAML::Parser> '' : value }
-# or
-#   {   ? !<tag:yaml.org,2002:perl/YAML::Parser:> value : '' }
-# Since we disallow non-empty node content, tags are always followed by spaces
-# or line breaks.
-
 # FIRST sets:
 # stream: { STREAM-START }
@@ -67 +50 @@
 from tokens import *
 from events import *
+from scanner import *
 
 class ParserError(MarkedYAMLError):
@@ -72 +56 @@
 
 class Parser:
-    # Since writing an LL(1) parser is a straightforward task, we do not give
-    # many comments here.
+    # Since writing a recursive-descendant parser is a straightforward task, we
+    # do not give many comments here.
     # Note that we use Python generators. If you rewrite the parser in another
     # language, you may replace all 'yield'-s with event handler calls.
@@ -82 +66 @@
     }
 
-    def __init__(self, scanner):
-        self.scanner = scanner
+    def __init__(self):
         self.current_event = None
         self.yaml_version = None
@@ -89 +72 @@
         self.event_generator = self.parse_stream()
 
-    def check(self, *choices):
+    def check_event(self, *choices):
         # Check the type of the next event.
         if self.current_event is None:
@@ -97 +80 @@
                 pass
         if self.current_event is not None:
+            if not choices:
+                return True
             for choice in choices:
                 if isinstance(self.current_event, choice):
@@ -102 +87 @@
         return False
 
-    def peek(self):
+    def peek_event(self):
         # Get the next event.
         if self.current_event is None:
@@ -111 +96 @@
         return self.current_event
 
-    def get(self):
+    def get_event(self):
         # Get the next event.
         if self.current_event is None:
@@ -130 +115 @@
 
         # Parse start of stream.
-        token = self.scanner.get()
+        token = self.get_token()
         yield StreamStartEvent(token.start_mark, token.end_mark,
                 encoding=token.encoding)
 
         # Parse implicit document.
-        if not self.scanner.check(DirectiveToken, DocumentStartToken,
+        if not self.check_token(DirectiveToken, DocumentStartToken,
                 StreamEndToken):
             self.tag_handles = self.DEFAULT_TAGS
-            token = self.scanner.peek()
+            token = self.peek_token()
             start_mark = end_mark = token.start_mark
             yield DocumentStartEvent(start_mark, end_mark,
@@ -144 +129 @@
             for event in self.parse_block_node():
                 yield event
-            token = self.scanner.peek()
+            token = self.peek_token()
             start_mark = end_mark = token.start_mark
             explicit = False
-            while self.scanner.check(DocumentEndToken):
-                token = self.scanner.get()
+            while self.check_token(DocumentEndToken):
+                token = self.get_token()
                 end_mark = token.end_mark
                 explicit = True
@@ -155 +140 @@
 
         # Parse explicit documents.
-        while not self.scanner.check(StreamEndToken):
-            token = self.scanner.peek()
+        while not self.check_token(StreamEndToken):
+            token = self.peek_token()
             start_mark = token.start_mark
             version, tags = self.process_directives()
-            if not self.scanner.check(DocumentStartToken):
+            if not self.check_token(DocumentStartToken):
                 raise ParserError(None, None,
                         "expected '<document start>', but found %r"
-                        % self.scanner.peek().id,
-                        self.scanner.peek().start_mark)
-            token = self.scanner.get()
+                        % self.peek_token().id,
+                        self.peek_token().start_mark)
+            token = self.get_token()
             end_mark = token.end_mark
             yield DocumentStartEvent(start_mark, end_mark,
                     explicit=True, version=version, tags=tags)
-            if self.scanner.check(DirectiveToken,
+            if self.check_token(DirectiveToken,
                     DocumentStartToken, DocumentEndToken, StreamEndToken):
                 yield self.process_empty_scalar(token.end_mark)
@@ -174 +159 @@
                 for event in self.parse_block_node():
                     yield event
-            token = self.scanner.peek()
+            token = self.peek_token()
             start_mark = end_mark = token.start_mark
             explicit = False
-            while self.scanner.check(DocumentEndToken):
-                token = self.scanner.get()
+            while self.check_token(DocumentEndToken):
+                token = self.get_token()
                 end_mark = token.end_mark
                 explicit=True
@@ -185 +170 @@
 
         # Parse end of stream.
-        token = self.scanner.get()
+        token = self.get_token()
         yield StreamEndEvent(token.start_mark, token.end_mark)
 
@@ -192 +177 @@
         self.yaml_version = None
         self.tag_handles = {}
-        while self.scanner.check(DirectiveToken):
-            token = self.scanner.get()
+        while self.check_token(DirectiveToken):
+            token = self.get_token()
             if token.name == u'YAML':
                 if self.yaml_version is not None:
@@ -238 +223 @@
         # block_node_or_indentless_sequence ::= ALIAS | properties?
         #                                       (block_content | indentless_block_sequence)
-        if self.scanner.check(AliasToken):
-            token = self.scanner.get()
+        if self.check_token(AliasToken):
+            token = self.get_token()
             yield AliasEvent(token.value, token.start_mark, token.end_mark)
         else:
@@ -245 +230 @@
             tag = None
             start_mark = end_mark = tag_mark = None
-            if self.scanner.check(AnchorToken):
-                token = self.scanner.get()
+            if self.check_token(AnchorToken):
+                token = self.get_token()
                 start_mark = token.start_mark
                 end_mark = token.end_mark
                 anchor = token.value
-                if self.scanner.check(TagToken):
-                    token = self.scanner.get()
+                if self.check_token(TagToken):
+                    token = self.get_token()
                     tag_mark = token.start_mark
                     end_mark = token.end_mark
                     tag = token.value
-            elif self.scanner.check(TagToken):
-                token = self.scanner.get()
+            elif self.check_token(TagToken):
+                token = self.get_token()
                 start_mark = tag_mark = token.start_mark
                 end_mark = token.end_mark
                 tag = token.value
-                if self.scanner.check(AnchorToken):
-                    token = self.scanner.get()
+                if self.check_token(AnchorToken):
+                    token = self.get_token()
                     end_mark = token.end_mark
                     anchor = token.value
-            if tag is not None:
+            if tag is not None and tag != u'!':
                 handle, suffix = tag
                 if handle is not None:
@@ -274 +259 @@
                 else:
                     tag = suffix
-            #if tag is None:
-            #    if not (self.scanner.check(ScalarToken) and
-            #            self.scanner.peek().implicit):
-            #        tag = u'!'
+            #if tag == u'!':
+            #    raise ParserError("while parsing a node", start_mark,
+            #            "found non-specific tag '!'", tag_mark,
+            #            "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
             if start_mark is None:
-                start_mark = end_mark = self.scanner.peek().start_mark
+                start_mark = end_mark = self.peek_token().start_mark
             event = None
             collection_events = None
-            if indentless_sequence and self.scanner.check(BlockEntryToken):
-                end_mark = self.scanner.peek().end_mark
+            if indentless_sequence and self.check_token(BlockEntryToken):
+                end_mark = self.peek_token().end_mark
                 event = SequenceStartEvent(anchor, tag, start_mark, end_mark)
                 collection_events = self.parse_indentless_sequence()
             else:
-                if self.scanner.check(ScalarToken):
-                    token = self.scanner.get()
+                if self.check_token(ScalarToken):
+                    token = self.get_token()
                     end_mark = token.end_mark
-                    implicit = (tag is None and token.implicit)
-                    event = ScalarEvent(anchor, tag, token.value,
-                            start_mark, end_mark,
-                            implicit=implicit, style=token.style)
-                elif self.scanner.check(FlowSequenceStartToken):
-                    end_mark = self.scanner.peek().end_mark
+                    implicit = ((tag is None or tag == u'!') and token.implicit)
+                    event = ScalarEvent(anchor, tag, implicit, token.value,
+                            start_mark, end_mark, style=token.style)
+                elif self.check_token(FlowSequenceStartToken):
+                    end_mark = self.peek_token().end_mark
                     event = SequenceStartEvent(anchor, tag, start_mark, end_mark,
                             flow_style=True)
                     collection_events = self.parse_flow_sequence()
-                elif self.scanner.check(FlowMappingStartToken):
-                    end_mark = self.scanner.peek().end_mark
+                elif self.check_token(FlowMappingStartToken):
+                    end_mark = self.peek_token().end_mark
                     event = MappingStartEvent(anchor, tag, start_mark, end_mark,
                             flow_style=True)
                     collection_events = self.parse_flow_mapping()
-                elif block and self.scanner.check(BlockSequenceStartToken):
-                    end_mark = self.scanner.peek().start_mark
+                elif block and self.check_token(BlockSequenceStartToken):
+                    end_mark = self.peek_token().start_mark
                     event = SequenceStartEvent(anchor, tag, start_mark, end_mark,
                             flow_style=False)
                     collection_events = self.parse_block_sequence()
-                elif block and self.scanner.check(BlockMappingStartToken):
-                    end_mark = self.scanner.peek().start_mark
+                elif block and self.check_token(BlockMappingStartToken):
+                    end_mark = self.peek_token().start_mark
                     event = MappingStartEvent(anchor, tag, start_mark, end_mark,
                             flow_style=False)
@@ -317 +301 @@
                     # Empty scalars are allowed even if a tag or an anchor is
                     # specified.
-                    event = ScalarEvent(anchor, tag, u'', start_mark, end_mark,
-                            implicit=True)
+                    implicit = (tag is None or tag == u'!')
+                    event = ScalarEvent(anchor, tag, implicit, u'',
+                            start_mark, end_mark)
                 else:
                     if block:
@@ -324 +309 @@
                     else:
                         node = 'flow'
-                    token = self.scanner.peek()
+                    token = self.peek_token()
                     raise ParserError("while scanning a %s node" % node, start_mark,
                             "expected the node content, but found %r" % token.id,
@@ -335 +320 @@
     def parse_block_sequence(self):
         # BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
-        token = self.scanner.get()
+        token = self.get_token()
         start_mark = token.start_mark
-        while self.scanner.check(BlockEntryToken):
-            token = self.scanner.get()
-            if not self.scanner.check(BlockEntryToken, BlockEndToken):
+        while self.check_token(BlockEntryToken):
+            token = self.get_token()
+            if not self.check_token(BlockEntryToken, BlockEndToken):
                 for event in self.parse_block_node():
                     yield event
             else:
                 yield self.process_empty_scalar(token.end_mark)
-        if not self.scanner.check(BlockEndToken):
-            token = self.scanner.peek()
+        if not self.check_token(BlockEndToken):
+            token = self.peek_token()
             raise ParserError("while scanning a block collection", start_mark,
                     "expected <block end>, but found %r" % token.id, token.start_mark)
-        token = self.scanner.get()
+        token = self.get_token()
         yield SequenceEndEvent(token.start_mark, token.end_mark)
 
     def parse_indentless_sequence(self):
         # (BLOCK-ENTRY block_node?)+
-        while self.scanner.check(BlockEntryToken):
-            token = self.scanner.get()
-            if not self.scanner.check(BlockEntryToken,
+        while self.check_token(BlockEntryToken):
+            token = self.get_token()
+            if not self.check_token(BlockEntryToken,
                     KeyToken, ValueToken, BlockEndToken):
                 for event in self.parse_block_node():
@@ -361 +346 @@
             else:
                 yield self.process_empty_scalar(token.end_mark)
-        token = self.scanner.peek()
+        token = self.peek_token()
         yield SequenceEndEvent(token.start_mark, token.start_mark)
 
@@ -369 +354 @@
         #   (VALUE block_node_or_indentless_sequence?)?)*
         # BLOCK-END
-        token = self.scanner.get()
+        token = self.get_token()
         start_mark = token.start_mark
-        while self.scanner.check(KeyToken, ValueToken):
-            if self.scanner.check(KeyToken):
-                token = self.scanner.get()
-                if not self.scanner.check(KeyToken, ValueToken, BlockEndToken):
+        while self.check_token(KeyToken, ValueToken):
+            if self.check_token(KeyToken):
+                token = self.get_token()
+                if not self.check_token(KeyToken, ValueToken, BlockEndToken):
                     for event in self.parse_block_node_or_indentless_sequence():
                         yield event
                 else:
                     yield self.process_empty_scalar(token.end_mark)
-            if self.scanner.check(ValueToken):
-                token = self.scanner.get()
-                if not self.scanner.check(KeyToken, ValueToken, BlockEndToken):
+            if self.check_token(ValueToken):
+                token = self.get_token()
+                if not self.check_token(KeyToken, ValueToken, BlockEndToken):
                     for event in self.parse_block_node_or_indentless_sequence():
                         yield event
@@ -387 +372 @@
                     yield self.process_empty_scalar(token.end_mark)
             else:
-                token = self.scanner.peek()
+                token = self.peek_token()
                 yield self.process_empty_scalar(token.start_mark)
-        if not self.scanner.check(BlockEndToken):
-            token = self.scanner.peek()
+        if not self.check_token(BlockEndToken):
+            token = self.peek_token()
             raise ParserError("while scanning a block mapping", start_mark,
                     "expected <block end>, but found %r" % token.id, token.start_mark)
-        token = self.scanner.get()
+        token = self.get_token()
         yield MappingEndEvent(token.start_mark, token.end_mark)
 
@@ -407 +392 @@
         # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
         # generate an inline mapping (set syntax).
-        token = self.scanner.get()
+        token = self.get_token()
         start_mark = token.start_mark
-        while not self.scanner.check(FlowSequenceEndToken):
-            if self.scanner.check(KeyToken):
-                token = self.scanner.get()
+        while not self.check_token(FlowSequenceEndToken):
+            if self.check_token(KeyToken):
+                token = self.get_token()
                 yield MappingStartEvent(None, None, # u'!',
                         token.start_mark, token.end_mark,
                         flow_style=True)
-                if not self.scanner.check(ValueToken,
+                if not self.check_token(ValueToken,
                         FlowEntryToken, FlowSequenceEndToken):
                     for event in self.parse_flow_node():
@@ -421 +406 @@
                 else:
                     yield self.process_empty_scalar(token.end_mark)
-                if self.scanner.check(ValueToken):
-                    token = self.scanner.get()
-                    if not self.scanner.check(FlowEntryToken, FlowSequenceEndToken):
+                if self.check_token(ValueToken):
+                    token = self.get_token()
+                    if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
                         for event in self.parse_flow_node():
                             yield event
@@ -429 +414 @@
                         yield self.process_empty_scalar(token.end_mark)
                 else:
-                    token = self.scanner.peek()
+                    token = self.peek_token()
                     yield self.process_empty_scalar(token.start_mark)
-                token = self.scanner.peek()
+                token = self.peek_token()
                 yield MappingEndEvent(token.start_mark, token.start_mark)
             else:
                 for event in self.parse_flow_node():
                     yield event
-            if not self.scanner.check(FlowEntryToken, FlowSequenceEndToken):
-                token = self.scanner.peek()
+            if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
+                token = self.peek_token()
                 raise ParserError("while scanning a flow sequence", start_mark,
                         "expected ',' or ']', but got %r" % token.id, token.start_mark)
-            if self.scanner.check(FlowEntryToken):
-                self.scanner.get()
-        token = self.scanner.get()
+            if self.check_token(FlowEntryToken):
+                self.get_token()
+        token = self.get_token()
         yield SequenceEndEvent(token.start_mark, token.end_mark)
 
@@ -451 +436 @@
         #                       FLOW-MAPPING-END
         # flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
-        token = self.scanner.get()
+        token = self.get_token()
         start_mark = token.start_mark
-        while not self.scanner.check(FlowMappingEndToken):
-            if self.scanner.check(KeyToken):
-                token = self.scanner.get()
-                if not self.scanner.check(ValueToken,
+        while not self.check_token(FlowMappingEndToken):
+            if self.check_token(KeyToken):
+                token = self.get_token()
+                if not self.check_token(ValueToken,
                         FlowEntryToken, FlowMappingEndToken):
                     for event in self.parse_flow_node():
@@ -462 +447 @@
                 else:
                     yield self.process_empty_scalar(token.end_mark)
-                if self.scanner.check(ValueToken):
-                    token = self.scanner.get()
-                    if not self.scanner.check(FlowEntryToken, FlowMappingEndToken):
+                if self.check_token(ValueToken):
+                    token = self.get_token()
+                    if not self.check_token(FlowEntryToken, FlowMappingEndToken):
                         for event in self.parse_flow_node():
                             yield event
@@ -470 +455 @@
                         yield self.process_empty_scalar(token.end_mark)
                 else:
-                    token = self.scanner.peek()
+                    token = self.peek_token()
                     yield self.process_empty_scalar(token.start_mark)
             else:
                 for event in self.parse_flow_node():
                     yield event
-                yield self.process_empty_scalar(self.scanner.peek().start_mark)
-            if not self.scanner.check(FlowEntryToken, FlowMappingEndToken):
-                token = self.scanner.peek()
+                yield self.process_empty_scalar(self.peek_token().start_mark)
+            if not self.check_token(FlowEntryToken, FlowMappingEndToken):
+                token = self.peek_token()
                 raise ParserError("while scanning a flow mapping", start_mark,
                         "expected ',' or '}', but got %r" % token.id, token.start_mark)
-            if self.scanner.check(FlowEntryToken):
-                self.scanner.get()
-        if not self.scanner.check(FlowMappingEndToken):
-            token = self.scanner.peek()
+            if self.check_token(FlowEntryToken):
+                self.get_token()
+        if not self.check_token(FlowMappingEndToken):
+            token = self.peek_token()
             raise ParserError("while scanning a flow mapping", start_mark,
                     "expected '}', but found %r" % token.id, token.start_mark)
-        token = self.scanner.get()
+        token = self.get_token()
         yield MappingEndEvent(token.start_mark, token.end_mark)
 
     def process_empty_scalar(self, mark):
-        return ScalarEvent(None, None, u'', mark, mark,
-                implicit=True)
-
+        return ScalarEvent(None, None, True, u'', mark, mark)
+

pyyaml/trunk/lib/yaml/reader.py

@@ -92 +92 @@
     # Yeah, it's ugly and slow.
 
-    def __init__(self, data):
+    def __init__(self, stream):
         self.name = None
         self.stream = None
@@ -105 +105 @@
         self.line = 0
         self.column = 0
-        if isinstance(data, unicode):
+        if isinstance(stream, unicode):
             self.name = "<unicode string>"
-            self.check_printable(data)
-            self.buffer = data+u'\0'
-        elif isinstance(data, str):
+            self.check_printable(stream)
+            self.buffer = stream+u'\0'
+        elif isinstance(stream, str):
             self.name = "<string>"
-            self.raw_buffer = data
+            self.raw_buffer = stream
             self.determine_encoding()
         else:
-            self.stream = data
-            self.name = getattr(data, 'name', "<file>")
+            self.stream = stream
+            self.name = getattr(stream, 'name', "<file>")
             self.eof = False
             self.raw_buffer = ''

pyyaml/trunk/lib/yaml/representer.py

@@ -21 +21 @@
     pass
 
-class BaseRepresenter(BaseDetector):
-
-    DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str'
-    DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq'
-    DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map'
-
-    def __init__(self, serializer):
-        self.serializer = serializer
+class BaseRepresenter:
+
+    yaml_representers = {}
+
+    def __init__(self):
         self.represented_objects = {}
 
-    def close(self):
-        self.serializer.close()
-
-    def represent(self, native):
-        node = self.represent_object(native)
-        self.serializer.serialize(node)
+    def represent(self, data):
+        node = self.represent_object(data)
+        self.serialize(node)
         self.represented_objects = {}
 
-    def represent_object(self, native):
-        if self.ignore_aliases(native):
+    def represent_object(self, data):
+        if self.ignore_aliases(data):
             alias_key = None
         else:
-            alias_key = id(native)
+            alias_key = id(data)
         if alias_key is not None:
             if alias_key in self.represented_objects:
                 node = self.represented_objects[alias_key]
                 if node is None:
-                    raise RepresenterError("recursive objects are not allowed: %r" % native)
+                    raise RepresenterError("recursive objects are not allowed: %r" % data)
                 return node
             self.represented_objects[alias_key] = None
-        for native_type in type(native).__mro__:
-            if native_type in self.yaml_representers:
-                node = self.yaml_representers[native_type](self, native)
+        for data_type in type(data).__mro__:
+            if data_type in self.yaml_representers:
+                node = self.yaml_representers[data_type](self, data)
                 break
         else:
             if None in self.yaml_representers:
-                node = self.yaml_representers[None](self, native)
+                node = self.yaml_representers[None](self, data)
             else:
-                node = ScalarNode(None, unicode(native))
+                node = ScalarNode(None, unicode(data))
         if alias_key is not None:
             self.represented_objects[alias_key] = node
         return node
 
-    def add_representer(cls, native_type, representer):
+    def add_representer(cls, data_type, representer):
         if not 'yaml_representers' in cls.__dict__:
             cls.yaml_representers = cls.yaml_representers.copy()
-        cls.yaml_representers[native_type] = representer
+        cls.yaml_representers[data_type] = representer
     add_representer = classmethod(add_representer)
 
-    yaml_representers = {}
-
     def represent_scalar(self, tag, value, style=None):
-        detected_tag = self.detect(value)
-        if detected_tag is None:
-            detected_tag = self.DEFAULT_SCALAR_TAG
-        implicit = (tag == detected_tag)
-        if tag == self.DEFAULT_SCALAR_TAG:
-            tag = None
-        return ScalarNode(tag, value, implicit=implicit, style=style)
+        return ScalarNode(tag, value, style=style)
 
     def represent_sequence(self, tag, sequence, flow_style=None):
-        if tag == self.DEFAULT_SEQUENCE_TAG:
-            tag = None
         value = []
         for item in sequence:
@@ -90 +74 @@
 
     def represent_mapping(self, tag, mapping, flow_style=None):
-        if tag == self.DEFAULT_MAPPING_TAG:
-            tag = None
         value = {}
         if hasattr(mapping, 'keys'):
@@ -104 +86 @@
         return MappingNode(tag, value, flow_style=flow_style)
 
-    def ignore_aliases(self, native):
+    def ignore_aliases(self, data):
         return False
 
-class SafeRepresenter(Detector, BaseRepresenter):
-
-    def ignore_aliases(self, native):
-        if native in [None, ()]:
+class SafeRepresenter(BaseRepresenter):
+
+    def ignore_aliases(self, data):
+        if data in [None, ()]:
             return True
-        if isinstance(native, (str, unicode, bool, int, float)):
+        if isinstance(data, (str, unicode, bool, int, float)):
             return True
 
-    def represent_none(self, native):
+    def represent_none(self, data):
         return self.represent_scalar(u'tag:yaml.org,2002:null',
                 u'null')
 
-    def represent_str(self, native):
+    def represent_str(self, data):
         encoding = None
         try:
-            unicode(native, 'ascii')
+            unicode(data, 'ascii')
             encoding = 'ascii'
         except UnicodeDecodeError:
             try:
-                unicode(native, 'utf-8')
+                unicode(data, 'utf-8')
                 encoding = 'utf-8'
             except UnicodeDecodeError:
@@ -132 +114 @@
         if encoding:
             return self.represent_scalar(u'tag:yaml.org,2002:str',
-                    unicode(native, encoding))
+                    unicode(data, encoding))
         else:
             return self.represent_scalar(u'tag:yaml.org,2002:binary',
-                    unicode(native.encode('base64')), style='|')
-
-    def represent_unicode(self, native):
-        return self.represent_scalar(u'tag:yaml.org,2002:str', native)
-
-    def represent_bool(self, native):
-        if native:
+                    unicode(data.encode('base64')), style='|')
+
+    def represent_unicode(self, data):
+        return self.represent_scalar(u'tag:yaml.org,2002:str', data)
+
+    def represent_bool(self, data):
+        if data:
             value = u'true'
         else:
@@ -147 +129 @@
         return self.represent_scalar(u'tag:yaml.org,2002:bool', value)
 
-    def represent_int(self, native):
-        return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(native))
-
-    def represent_long(self, native):
-        return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(native))
+    def represent_int(self, data):
+        return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data))
+
+    def represent_long(self, data):
+        return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data))
 
     inf_value = 1e300000
     nan_value = inf_value/inf_value
 
-    def represent_float(self, native):
-        if native == self.inf_value:
+    def represent_float(self, data):
+        if data == self.inf_value:
             value = u'.inf'
-        elif native == -self.inf_value:
+        elif data == -self.inf_value:
             value = u'-.inf'
-        elif native == self.nan_value or native != native:
+        elif data == self.nan_value or data != data:
             value = u'.nan'
         else:
-            value = unicode(native)
+            value = unicode(data)
         return self.represent_scalar(u'tag:yaml.org,2002:float', value)
 
-    def represent_list(self, native):
-        pairs = (len(native) > 0)
-        for item in native:
+    def represent_list(self, data):
+        pairs = (len(data) > 0)
+        for item in data:
             if not isinstance(item, tuple) or len(item) != 2:
                 pairs = False
                 break
         if not pairs:
-            return self.represent_sequence(u'tag:yaml.org,2002:seq', native)
+            return self.represent_sequence(u'tag:yaml.org,2002:seq', data)
         value = []
-        for item_key, item_value in native:
+        for item_key, item_value in data:
             value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
                 [(item_key, item_value)]))
         return SequenceNode(u'tag:yaml.org,2002:pairs', value)
 
-    def represent_dict(self, native):
-        return self.represent_mapping(u'tag:yaml.org,2002:map', native)
-
-    def represent_set(self, native):
+    def represent_dict(self, data):
+        return self.represent_mapping(u'tag:yaml.org,2002:map', data)
+
+    def represent_set(self, data):
         value = {}
-        for key in native:
+        for key in data:
             value[key] = None
         return self.represent_mapping(u'tag:yaml.org,2002:set', value)
 
-    def represent_date(self, native):
-        value = u'%04d-%02d-%02d' % (native.year, native.month, native.day)
+    def represent_date(self, data):
+        value = u'%04d-%02d-%02d' % (data.year, data.month, data.day)
         return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value)
 
-    def represent_datetime(self, native):
+    def represent_datetime(self, data):
         value = u'%04d-%02d-%02d %02d:%02d:%02d' \
-                % (native.year, native.month, native.day,
-                    native.hour, native.minute, native.second)
-        if native.microsecond:
-            value += u'.' + unicode(native.microsecond/1000000.0).split(u'.')[1]
-        if native.utcoffset():
-            value += unicode(native.utcoffset())
+                % (data.year, data.month, data.day,
+                    data.hour, data.minute, data.second)
+        if data.microsecond:
+            value += u'.' + unicode(data.microsecond/1000000.0).split(u'.')[1]
+        if data.utcoffset():
+            value += unicode(data.utcoffset())
         return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value)
 
-    def represent_undefined(self, native):
-        raise RepresenterError("cannot represent an object: %s" % native)
+    def represent_yaml_object(self, tag, data, cls, flow_style=None):
+        if hasattr(data, '__getstate__'):
+            state = data.__getstate__()
+        else:
+            state = data.__dict__.copy()
+        mapping = state
+        if hasattr(state, 'keys'):
+            mapping = []
+            keys = state.keys()
+            keys.sort()
+            for key in keys:
+                mapping.append((key.replace('_', '-'), state[key]))
+        return self.represent_mapping(tag, mapping, flow_style=flow_style)
+
+    def represent_undefined(self, data):
+        raise RepresenterError("cannot represent an object: %s" % data)
 
 SafeRepresenter.add_representer(type(None),

pyyaml/trunk/lib/yaml/scanner.py

@@ -46 +46 @@
 class Scanner:
 
-
-    def __init__(self, reader):
+    def __init__(self):
         """Initialize the scanner."""
-        # The input stream. The Reader class do the dirty work of checking for
-        # BOM and converting the input data to Unicode. It also adds NUL to
-        # the end.
+        # It is assumed that Scanner and Reader will have a common descendant.
+        # Reader do the dirty work of checking for BOM and converting the
+        # input data to Unicode. It also adds NUL to the end.
         #
         # Reader supports the following methods
-        #   self.reader.peek(i=0)       # peek the next i-th character
-        #   self.reader.prefix(l=1)     # peek the next l characters
-        #   self.reader.forward(l=1)    # read the next l characters
-                                        # and move the pointer
-        self.reader = reader
+        #   self.peek(i=0)       # peek the next i-th character
+        #   self.prefix(l=1)     # peek the next l characters
+        #   self.forward(l=1)    # read the next l characters and move the pointer.
 
         # Had we reached the end of the stream?
@@ -114 +111 @@
     # Public methods.
 
-    def check(self, *choices):
+    def check_token(self, *choices):
         # Check if the next token is one of the given types.
         while self.need_more_tokens():
             self.fetch_more_tokens()
         if self.tokens:
+            if not choices:
+                return True
             for choice in choices:
                 if isinstance(self.tokens[0], choice):
@@ -124 +123 @@
         return False
 
-    def peek(self):
+    def peek_token(self):
         # Return the next token, but do not delete if from the queue.
         while self.need_more_tokens():
@@ -131 +130 @@
             return self.tokens[0]
 
-    def get(self):
+    def get_token(self):
         # Return the next token.
         while self.need_more_tokens():
@@ -172 +171 @@
         # Compare the current indentation and column. It may add some tokens
         # and decrease the current indentation level.
-        self.unwind_indent(self.reader.column)
+        self.unwind_indent(self.column)
 
         # Peek the next character.
-        ch = self.reader.peek()
+        ch = self.peek()
 
         # Is it the end of stream?
@@ -266 +265 @@
         raise ScannerError("while scanning for the next token", None,
                 "found character %r that cannot start any token"
-                % ch.encode('utf-8'), self.reader.get_mark())
+                % ch.encode('utf-8'), self.get_mark())
 
     # Simple keys treatment.
@@ -294 +293 @@
         for level in self.possible_simple_keys.keys():
             key = self.possible_simple_keys[level]
-            if key.line != self.reader.line  \
-                    or self.reader.index-key.index > 1024:
+            if key.line != self.line  \
+                    or self.index-key.index > 1024:
                 if key.required:
                     raise ScannerError("while scanning a simple key", key.mark,
-                            "could not found expected ':'", self.reader.get_mark())
+                            "could not found expected ':'", self.get_mark())
                 del self.possible_simple_keys[level]
 
@@ -307 +306 @@
 
         # Check if a simple key is required at the current position.
-        required = not self.flow_level and self.indent == self.reader.column
+        required = not self.flow_level and self.indent == self.column
 
         # A simple key is required only if it is the first token in the current
@@ -318 +317 @@
             self.remove_possible_simple_key()
             token_number = self.tokens_taken+len(self.tokens)
-            index = self.reader.index
-            line = self.reader.line
-            column = self.reader.column
-            mark = self.reader.get_mark()
             key = SimpleKey(token_number, required,
-                    index, line, column, mark)
+                    self.index, self.line, self.column, self.get_mark())
             self.possible_simple_keys[self.flow_level] = key
 
@@ -335 +330 @@
             #if key.required:
             #    raise ScannerError("while scanning a simple key", key.mark,
-            #            "could not found expected ':'", self.reader.get_mark())
+            #            "could not found expected ':'", self.get_mark())
 
     # Indentation functions.
@@ -350 +345 @@
         #    raise ScannerError(None, None,
         #            "invalid intendation or unclosed '[' or '{'",