Module: Pocolog
- Defined in:
- lib/pocolog.rb,
lib/pocolog/format.rb,
lib/pocolog/logger.rb,
lib/pocolog/repair.rb,
lib/pocolog/convert.rb,
lib/pocolog/upgrade.rb,
lib/pocolog/version.rb,
lib/pocolog/logfiles.rb,
lib/pocolog/format/v2.rb,
lib/pocolog/data_stream.rb,
lib/pocolog/io_sequence.rb,
lib/pocolog/stream_info.rb,
lib/pocolog/upgrade/dsl.rb,
lib/pocolog/block_stream.rb,
lib/pocolog/stream_index.rb,
lib/pocolog/test_helpers.rb,
lib/pocolog/file_upgrader.rb,
lib/pocolog/stream_aligner.rb,
lib/pocolog/warn_deprecated.rb,
lib/pocolog/cli/tty_reporter.rb,
lib/pocolog/upgrade/ops/base.rb,
lib/pocolog/cli/null_reporter.rb,
lib/pocolog/sample_enumerator.rb,
lib/pocolog/file_index_builder.rb,
lib/pocolog/upgrade/exceptions.rb,
lib/pocolog/upgrade/ops/custom.rb,
lib/pocolog/upgrade/ops/identity.rb,
lib/pocolog/upgrade/ops/sequence.rb,
lib/pocolog/upgrade/ops/enum_cast.rb,
lib/pocolog/upgrade/ops/array_cast.rb,
lib/pocolog/io_sequence_stream_info.rb,
lib/pocolog/upgrade/ops/numeric_cast.rb,
lib/pocolog/upgrade/ops/compound_cast.rb,
lib/pocolog/upgrade/converter_registry.rb,
lib/pocolog/upgrade/ops/container_cast.rb
Defined Under Namespace
Modules: CLI, Format, TestHelpers, Upgrade Classes: BlockStream, DataStream, FileUpgrader, IOSequence, IndexBuilderStreamInfo, InternalError, InvalidBlockFound, InvalidFile, InvalidIndex, Logfiles, Logger, MissingIndexPrologue, MissingPrologue, NotEnoughData, ObsoleteIndexVersion, ObsoleteVersion, SampleEnumerator, StreamAligner, StreamIndex, StreamInfo
Constant Summary
- STREAM_BLOCK =
1
- DATA_BLOCK =
2
- CONTROL_BLOCK =
3
- BLOCK_TYPES =
[STREAM_BLOCK, DATA_BLOCK, CONTROL_BLOCK]
- DATA_STREAM =
1
- CONTROL_SET_TIMEBASE =
0
- CONTROL_SET_TIMEOFFSET =
1
- VERSION =
"0.4"
Class Method Summary collapse
-
.big_endian? ⇒ Boolean
true if this machine is big endian.
-
.create_index_from_raw_info(block_stream, raw_info) ⇒ Array<StreamInfo,nil>
Create an list of StreamInfo object based on basic information about the file.
-
.file_index_builder(block_stream) ⇒ Array<StreamInfo,nil>
Build the index of a block stream.
-
.file_stream(file_name, stream_name) ⇒ Object
Returns the stream called
stream_name
from file. -
.find_valid_range(block_stream) ⇒ Object
Find the byte position just after the last valid block in the stream.
-
.repair_file(path, keep_invalid_files: true, backup: true, reporter: CLI::NullReporter.new) ⇒ Boolean
Attempt to repair a broken file.
- .show_deprecations=(flag) ⇒ Object
- .show_deprecations? ⇒ Boolean
-
.warn_deprecated(message) ⇒ Object
Warns about a deprecation, showing the first backtrace lines.
Instance Method Summary collapse
-
#index_map ⇒ Array<(Integer,Integer,Integer)>
(block_pos,lg_time,sample_index) tuples.
-
#stream_block_pos ⇒ Integer
The position of the stream definition in the IO.
Class Method Details
.big_endian? ⇒ Boolean
true if this machine is big endian
32 33 34 |
# File 'lib/pocolog.rb', line 32 def self.big_endian? "LAAS".unpack('L').pack('N') == "LAAS" end |
.create_index_from_raw_info(block_stream, raw_info) ⇒ Array<StreamInfo,nil>
Create an list of StreamInfo object based on basic information about the file
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
# File 'lib/pocolog/file_index_builder.rb', line 53 def self.create_index_from_raw_info(block_stream, raw_info) raw_info.map do |stream_info| next unless stream_info index_map = stream_info.index_map interval_rt = [] base_time = nil # Read the realtime of the first and last samples unless index_map.empty? block_stream.seek(index_map.first) block_stream.read_next_block_header first_block = block_stream.read_data_block_header block_stream.seek(index_map[-2]) block_stream.read_next_block_header last_block = block_stream.read_data_block_header interval_rt = [first_block.rt_time, last_block.rt_time] base_time = index_map[1] sample_index = -1 index_map = index_map.each_slice(2).map do |block_pos, lg_time| sample_index += 1 [block_pos, lg_time - base_time, sample_index] end end StreamInfo.from_raw_data( stream_info.stream_block_pos, interval_rt, base_time, index_map ) end end |
.file_index_builder(block_stream) ⇒ Array<StreamInfo,nil>
Build the index of a block stream
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
# File 'lib/pocolog/file_index_builder.rb', line 21 def self.file_index_builder(block_stream) # We build the information expected by StreamInfo.from_raw_data # That is (declaration_block, interval_rt, base_time, index_map) # # The first pass extracts information from the file itself, but # avoiding as much tests-in-the-loop as possible. This array # therefore only stores the declaration block and index map raw_stream_info = [] block_pos = block_stream.tell while (block = block_stream.read_next_block_header) stream_index = block.stream_index if block.kind == STREAM_BLOCK raw_stream_info[stream_index] = IndexBuilderStreamInfo.new(block_pos, []) elsif block.kind == DATA_BLOCK data_block = block_stream.read_data_block_header index_map = raw_stream_info[stream_index].index_map index_map << block_pos << data_block.lg_time end block_stream.skip_payload block_pos = block_stream.tell end create_index_from_raw_info(block_stream, raw_stream_info) end |
.file_stream(file_name, stream_name) ⇒ Object
Returns the stream called stream_name
from file
656 657 658 659 |
# File 'lib/pocolog/logfiles.rb', line 656 def self.file_stream(file_name, stream_name) file = Logfiles.open(file_name) file.stream(stream_name) end |
.find_valid_range(block_stream) ⇒ Object
Find the byte position just after the last valid block in the stream
It starts the search at the stream's current position
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
# File 'lib/pocolog/repair.rb', line 69 def self.find_valid_range(block_stream) stream_sample = {} current_pos = block_stream.tell while (block = block_stream.read_next_block_header) if block.kind == STREAM_BLOCK stream = block_stream.read_stream_block stream_sample[block.stream_index] = stream.type.new elsif block.kind == DATA_BLOCK _, marshalled_data = block_stream.read_data_block stream_sample[block.stream_index] .from_buffer_direct(marshalled_data) current_pos = block_stream.tell end end current_pos rescue InvalidFile current_pos end |
.repair_file(path, keep_invalid_files: true, backup: true, reporter: CLI::NullReporter.new) ⇒ Boolean
Attempt to repair a broken file
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
# File 'lib/pocolog/repair.rb', line 9 def self.repair_file( path, keep_invalid_files: true, backup: true, reporter: CLI::NullReporter.new ) size = File.stat(path).size reporter.('[:bar]', total: size) block_stream = BlockStream.open(path) broken_path = "#{path}.broken" if backup begin block_stream.read_prologue rescue InvalidFile reporter.error "#{path}: missing or invalid prologue, nothing to salvage" return false if keep_invalid_files if broken_path FileUtils.mv path, broken_path reporter.error "moved #{path} to #{broken_path}" else FileUtils.rm_f path reporter.error "deleted #{path}" end return false end stream_info = {} stream_sample = {} current_pos = block_stream.tell begin while (block = block_stream.read_next_block_header) if block.kind == STREAM_BLOCK stream = block_stream.read_stream_block stream_info[block.stream_index] = stream stream_sample[block.stream_index] = stream.type.new else _, marshalled_data = block_stream.read_data_block stream_sample[block.stream_index] .from_buffer_direct(marshalled_data) end current_pos = block_stream.tell reporter.current = current_pos end reporter.finish true rescue InvalidFile reporter.finish remaining = File.stat(path).size - current_pos reporter.error "#{path}: broken at position #{current_pos} "\ "(#{remaining} bytes thrown away)" reporter.error " copying the current file as #{path}.broken" FileUtils.cp path, "#{path}.broken" reporter.error ' truncating the existing file' File.open(path, 'a') { |io| io.truncate(current_pos) } false end end |
.show_deprecations=(flag) ⇒ Object
6 7 8 |
# File 'lib/pocolog/warn_deprecated.rb', line 6 def self.show_deprecations=(flag) @show_deprecations = flag end |
.show_deprecations? ⇒ Boolean
2 3 4 |
# File 'lib/pocolog/warn_deprecated.rb', line 2 def self.show_deprecations? @show_deprecations end |
.warn_deprecated(message) ⇒ Object
Warns about a deprecation, showing the first backtrace lines
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
# File 'lib/pocolog/warn_deprecated.rb', line 13 def self.warn_deprecated() return if !show_deprecations? # Show the message, regardless of the actual logger setup current_logger_level = logger.level logger.level = Logger::WARN warn caller(1)[0, 4].each do |line| warn " #{line}" end ensure if current_logger_level logger.level = current_logger_level end end |
Instance Method Details
#index_map ⇒ Array<(Integer,Integer,Integer)>
(block_pos,lg_time,sample_index) tuples. The logical time is in absolute microseconds.
14 |
# File 'lib/pocolog/file_index_builder.rb', line 14 IndexBuilderStreamInfo = Struct.new :stream_block_pos, :index_map |
#stream_block_pos ⇒ Integer
Returns the position of the stream definition in the IO
14 |
# File 'lib/pocolog/file_index_builder.rb', line 14 IndexBuilderStreamInfo = Struct.new :stream_block_pos, :index_map |