Datachain

McapFile

McapFile is a File holding an MCAP robotics log. McapFile and McapSlice rows are created by read_mcap.

McapFile

McapFile(**kwargs)

Bases: File

File holding an MCAP robotics log (ROS 2, Foxglove).

Source code in datachain/lib/file.py
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
def __init__(self, **kwargs):
    super().__init__(**kwargs)
    # Annotations only: in the class body the pydantic mypy plugin would take
    # them for fields.
    self._caching_enabled: bool
    self._download_cb: Callback
    self._fs_path_cache: tuple[str, str, str] | None
    # The runtime state, set in one __dict__ update: pydantic's __setattr__ per
    # attribute is slow when millions of Files are built. A subclass that
    # declares them as private attributes needs pydantic's setattr instead.
    if _FILE_STATE.keys() & self.__private_attributes__.keys():
        for name, value in _FILE_STATE.items():
            setattr(self, name, value)
    else:
        self.__dict__.update(_FILE_STATE)

messages

messages(topics: str | Sequence[str] | None = None, start: int | None = None, end: int | None = None, decoder_factories: Sequence[Any] = ()) -> Iterator[McapMessage]

Yield messages in log-time order, logged in [start, end) nanoseconds.

Only the chunks that may hold matching messages are read. With decoder_factories (e.g. mcap_protobuf.decoder.DecoderFactory()), each message also carries its decoded object.

Source code in datachain/lib/mcap.py
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def messages(
    self,
    topics: str | Sequence[str] | None = None,
    start: int | None = None,
    end: int | None = None,
    decoder_factories: Sequence[Any] = (),
) -> Iterator[McapMessage]:
    """Yield messages in log-time order, logged in `[start, end)` nanoseconds.

    Only the chunks that may hold matching messages are read. With
    `decoder_factories` (e.g. `mcap_protobuf.decoder.DecoderFactory()`), each
    message also carries its `decoded` object.
    """
    window = start is not None or end is not None
    with self.open(block_size=PARTIAL_READ_BLOCK_SIZE if window else None) as f:
        reader = _reader(f, decoder_factories)
        names = _topic_list(topics)
        args = {"topics": names, "starting_at": start, "ending_before": end}
        rows = (
            reader.iter_decoded_messages(**args)
            if decoder_factories
            else ((*row, None) for row in reader.iter_messages(**args))
        )
        for schema, channel, msg, decoded in rows:
            message = McapMessage(
                topic=channel.topic,
                log_time=msg.log_time,
                publish_time=msg.publish_time,
                data=msg.data,
                message_encoding=channel.message_encoding,
                **_schema_fields(schema),
            )
            message._decoded = decoded
            yield message

summary

summary(topics: str | Sequence[str] | None = None) -> McapSummary

Read the summary section only, optionally counting just topics.

Source code in datachain/lib/mcap.py
80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
def summary(self, topics: str | Sequence[str] | None = None) -> McapSummary:
    """Read the summary section only, optionally counting just `topics`."""
    topics = _topic_list(topics)
    with self._open_read(block_size=PARTIAL_READ_BLOCK_SIZE, cache=False) as f:
        s = _reader(f).get_summary()
    st = s and s.statistics
    if st is None or (st.channel_count, st.schema_count) != (
        len(s.channels),
        len(s.schemas),
    ):
        raise FileError("MCAP file has no complete summary", self.source, self.path)
    counts = st.channel_message_counts
    channels = [
        McapTopic(
            topic=ch.topic,
            messages=counts.get(ch.id, 0),
            message_encoding=ch.message_encoding,
            **_schema_fields(s.schemas.get(ch.schema_id)),
        )
        for ch in s.channels.values()
        if topics is None or ch.topic in topics
    ]
    start, end = st.message_start_time, st.message_end_time
    return McapSummary(
        topics=channels,
        messages=sum(t.messages for t in channels),
        start=start,
        end=end,
        duration=(end - start) / 1e9,
    )

McapSlice

Bases: DataModel

A [start, end) log-time window of an MCAP file, in nanoseconds.

Attributes:

  • mcap (McapFile) –

    The MCAP file.

  • start (int) –

    Window start, inclusive.

  • end (int) –

    Window end, exclusive.

  • topics (list[str]) –

    Topics messages() returns by default; empty means all.

messages

messages(topics: str | Sequence[str] | None = None, decoder_factories: Sequence[Any] = ()) -> Iterator[McapMessage]

Yield the window's messages, of topics or else of self.topics.

Source code in datachain/lib/mcap.py
162
163
164
165
166
167
168
169
170
def messages(
    self,
    topics: str | Sequence[str] | None = None,
    decoder_factories: Sequence[Any] = (),
) -> Iterator[McapMessage]:
    """Yield the window's messages, of `topics` or else of `self.topics`."""
    if topics is None:
        topics = self.topics or None
    return self.mcap.messages(topics, self.start, self.end, decoder_factories)

McapSummary

Bases: DataModel

Totals from the summary section of an MCAP file.

Attributes:

  • topics (list[McapTopic]) –

    One entry per channel.

  • messages (int) –

    Message count over topics.

  • start (int) –

    Log time of the first message in the file, in nanoseconds.

  • end (int) –

    Log time of the last message in the file, in nanoseconds.

  • duration (float) –

    end - start, in seconds.

McapTopic

Bases: DataModel

A channel of an MCAP file: its topic, message count and encodings.

McapMessage

Bases: DataModel

An MCAP message; times are in nanoseconds.

decoded holds the decoded message when messages() got decoder_factories; it isn't saved with the dataset.