package App::JsonLogUtils;
# ABSTRACT: Command line utilities for dealing with JSON-formatted log files
$App::JsonLogUtils::VERSION = '0.03';

use strict;
use warnings;

use Fcntl             qw(:seek);
use Iterator::Simple  qw(iterator iter igrep imap ichain);
use JSON::XS          qw(decode_json encode_json);
use Time::HiRes       qw(sleep);
use Term::SimpleColor;

use parent 'Exporter';

our @EXPORT_OK = qw(

# Internal utilities
sub log_warn { warn red,    @_, default, "\n" }
sub log_info { warn yellow, @_, default, "\n" }

sub _open {
  my $path = shift || return;
  return $path if ref $path;

  open(my $fh, '<', $path) || return do{
    log_warn $!;

  return $fh;

sub lines ($) {
  my $path = shift;
  my $fh   = _open($path) || return iter([]);
  imap{ chomp $_; $_ } iter $fh;

sub tail ($) {
  my $path = shift;
  my $fh   = _open($path) || return iter([]);
  my $pos  = 0;
  my $stop = 0;

  seek $fh, 0, SEEK_END;
  $pos = tell $fh;

  $SIG{INT} = sub{
    log_info 'Stopped';
    $stop = 1;

      # Check for control-c
      if ($stop) {
        undef $SIG{INT};

      # Check for file truncation
      my $eof = eof $fh;
      my $cur = tell $fh;

      seek $fh, 0, SEEK_END;
      my $end = tell $fh;

      if ($end < $cur) {
        log_info 'File truncated';
        $pos = $end;
      else {
        $pos = $cur;

      seek $fh, $pos, SEEK_SET;
      <$fh> if $eof;

      # Return next line
      if (defined(my $line = <$fh>)) {
        chomp $line;
        return $line;

      # Reset position
      seek $fh, $pos, SEEK_SET;

      # Reset EOF condition on handle and wait for new input
      seek $fh, 0, SEEK_CUR;
      sleep 0.2;

      # Try again
      goto LINE;

sub json_log ($) {
  my $lines = shift;

    while (defined(my $line = <$lines>)) {
      if (!$line) {
        log_info 'empty line';

      my $obj = eval{ decode_json $line };

      if ($@) {
        log_warn "invalid JSON: $line";

      return [$obj, $line];


sub json_cols ($$$) {
  my ($cols, $sep, $lines) = @_;
  my @cols = ref $cols ? @$cols : split /\s+/, $cols;
  my $head = iter [ join($sep, @cols) ];
  my $rows = imap{
    my $obj = $_->[0];
    return join($sep, map{ $obj->{$_} || '' } @cols);
  } json_log $lines;
  ichain $head, $rows;

sub json_cut ($$$) {
  my ($fields, $inverse, $lines) = @_;
  my @fields = ref $fields ? @$fields : split /\s+/, $fields;

  if ($inverse) {
      foreach my $field (@fields) {
        delete $_->[0]{$field};

    } json_log $lines;
  else {
      my %filtered;
      foreach my $field (@fields) {
        $filtered{$field} = $_->[0]{$field};

    } json_log $lines;

sub json_grep ($$$) {
  my ($patterns, $inverse, $lines) = @_;
  return igrep{
    my $obj = $_->[0];

    foreach my $field (keys %$patterns) {
      foreach my $pattern (@{$patterns->{$field}}) {
        return unless $inverse
          ? $obj->{$field} !~ $pattern
          : $obj->{$field} =~ $pattern;

    return 1;
  json_log $lines;




=encoding UTF-8

=head1 NAME

App::JsonLogUtils - Command line utilities for dealing with JSON-formatted log files

=head1 VERSION

version 0.03


  # From the command line
  tail -f /path/to/log/file.log \
    | jgrep -m message="some pattern" \
    | jcut -f "timestamp priority message" \
    | cols -c "timestamp priority message" -s '|' \
    | column -t -s '|'

  # From code
  use App::JsonLogUtils qw(tail json_log);

  my $log = json_log tail '/path/to/file.log';

  while (my $entry = <$log>) {
    my ($json, $line) = @$entry;

  # Grepping JSON logs
  use App::JsonLogUtils qw(lines json_log);
  use Iterator::Simple qw(igrep imap);

  my $entries = igrep{ $_->{foo} =~ /bar/ } # filter objects
                imap{ $_->[0] }             # select the object
                json_log                    # parse
                lines '/path/to/file.log';  # read


Writing logs in JSON, one object per line, makes them very easily machine
readable. Wonderful. Unfortunately, it also makes it unfuriating to deal with
them using the standard unix command line tools. This package provides a few
tools to salve the burn.


=head2 L<jgrep>

Greps patterns in individual object fields.

=head2 L<jcut>

Filter the fields included in objects.

=head2 L<jcols>

Display fields in a format suitable for C<column>.

=head2 L<jshell>

An interactive shell for monitoring JSON log files.


If desired, the iterators used to implement the tools above are optionally
exported by the main module.

=head1 lines

Accepts a file path or opened file handle and returns an iterator which yields
the chomped lines from the file.

  my $log = lines '/path/to/file.log';

  while (my $line = <$log>) {

=head1 tail

Accepts a file path or opened file handle and returns an iterator while yields
chomped lines from the file as they are appended, starting from the end of the
file. Lines already written to the file when this routine is first called are
ignored (that is, there is no equivalent to C<tail -c 10> at this time).

  my $log = tail '/path/to/file.log';

  while (my $line = <$log>) { # sleeps until lines appended to file

=head1 json_log

Accepts a file iterator (see L</tail> and L</lines>) and returns an iterator
yielding an array ref holding two items, a hash ref of the parsed JSON log
entry, and the original log entry string. Empty lines are skipped with a
warning. JSON decoding errors are ignored with a warning.

  my $lines = json_log tail '/path/to/file.log';

  while (my $entry = <$lines>) {
    my ($object, $line) = @_;

=head2 json_cols

Accepts a list of fields (as a space-separared string or array ref of strings),
a string separator, and an iterator over JSON object strings, and returns a new
iterator. The returned iterator will first yield a string of column names
joined by the separator string. Subsequent calls will iterate over the JSON
object strings, return the value of each of the selected fields joined by the
separator string.

  # File $input
  {"a": 1, "b": 2, "c": 3}
  {"a": 4, "b": 5, "c": 6}
  {"a": 7, "b": 8, "c": 9}

  # Select columns a and c, separated by a pipe
  my $cols = json_cols "a c", "|" , lines $input;

  # ...yields the following strings:

=head2 json_cut

Accepts a space-separated string or array ref of C<$fields>, boolean
C<$inverse>, and an iterator of JSON log lines. Returns an iterator yielding
objects containing only the fields selected in C<$fields>. If C<$inverse> is
true, instead yields objects containing only the fields I<not> contained in

Give the same input as the L<previous example|/json_cols>:

  my $cut = json_cut "a c", 0, lines $input;

  # ...yields the following hash refs:
  {a => 1, c => 3}
  {a => 4, c => 6}
  {a => 7, c => 9}

  # Inverted
  my $cut = json_cut "a c", 1, lines $input;

  # ...yields:
  {b => 2}
  {b => 5}
  {b => 8}

=head2 json_grep

Accepts a hash ref where keys are field names and values are arrays of
regular expressions, a boolean C<$inverse>, and an iterator of JSON object
strings. Returns an iterator yielding array refs of the parsed JSON object
hash and the original string (just like L</json_log>). Only those entries
for which all fields' patterns match are returned. If C<$inverse> is set,
the logic is negated and only those entries for which all patterns test
false are returned.

  # File $input
  {"foo": "bar"}
  {"foo": "baz"}
  {"foo": "bat"}
  {"foo": "BAR"}

  # Code
  my $entries = json_grep { foo => [qw/bar/i, qr/baz/] }, 0, lines $input;

  # ...yields the following:
  [ {foo => "bar"}, '{"foo": "bar"}' ]
  [ {foo => "baz"}, '{"foo": "baz"}' ]
  [ {foo => "BAR"}, '{"foo": "BAR"}' ]


None, but will happily consider requests and patches.

=head1 AUTHOR

Jeff Ober <>


This software is copyright (c) 2018 by Jeff Ober.

This is free software; you can redistribute it and/or modify it under
the same terms as the Perl 5 programming language system itself.