Skip to content

Commit

Permalink
Merge pull request #23 from tclahr/v1.6.0
Browse files Browse the repository at this point in the history
V1.6.0
  • Loading branch information
tclahr authored Jul 24, 2021
2 parents db68007 + bf85087 commit 6cddc9a
Show file tree
Hide file tree
Showing 14 changed files with 258 additions and 308 deletions.
56 changes: 36 additions & 20 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,22 @@
# Changelog
All notable changes to this project will be documented in this file.

## 1.6.0 (2021-07-24)

### Added
- logs, system_files, user_files and suspicious_files will now be stored into a single compressed file (files.tar.gz).
- New entries added to system_files.conf
- /private/var/spool
- New docker_virtual_machine collectors
- podman container ls --all --size
- podman image ls --all
- podman info
- podman container logs <ID>
- podman inspect <ID>
- podman network inspect <ID>
- podman top <ID>
- podman version

## 1.5.1 (2021-06-07)

### Added
Expand Down Expand Up @@ -74,26 +90,26 @@ All notable changes to this project will be documented in this file.
- Files and directories added to ```conf/exclude.conf``` will be skipped during collection.
- By default, mounted remote file systems will be excluded from the collection. Please refer to ```conf/uac.conf``` for more information.
- New docker_virtual_machine collectors
- docker container ls --all --size
- docker image ls --all
- docker info
- docker inspect <ID>
- docker network inspect <ID>
- docker top <ID>
- docker version
- docker container logs
- virsh list --all
- virsh domifaddr <NAME>
- virsh dominfo <NAME>
- virsh dommemstat <NAME>
- virsh snapshot-list <NAME>
- virsh vcpuinfo <DOMAIN>
- virsh net-list --all
- virsh net-info <NAME>
- virsh net-dhcp-leases <NAME>
- virsh nodeinfo
- virsh pool-list --all
- virt-top -n 1
- docker container ls --all --size
- docker image ls --all
- docker info
- docker container logs <ID>
- docker inspect <ID>
- docker network inspect <ID>
- docker top <ID>
- docker version
- virsh list --all
- virsh domifaddr <NAME>
- virsh dominfo <NAME>
- virsh dommemstat <NAME>
- virsh snapshot-list <NAME>
- virsh vcpuinfo <DOMAIN>
- virsh net-list --all
- virsh net-info <NAME>
- virsh net-dhcp-leases <NAME>
- virsh nodeinfo
- virsh pool-list --all
- virt-top -n 1
- New process collectors
- ps -eo pid,etime,args
- ps -eo pid,lstart,args
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.5.1
1.6.0
11 changes: 10 additions & 1 deletion collectors/docker_virtual_machine
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (C) 2019,2020 IBM Corporation
# Copyright (C) 2020 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
Expand All @@ -19,10 +19,19 @@ logging I ">> Collecting docker and virtual machines information\n"
run_command "docker container ls --all --size" "$COLLECTOR_OUTPUT_DIR/docker-container-ls-all-size.txt"
run_command "docker image ls --all" "$COLLECTOR_OUTPUT_DIR/docker-image-ls-all.txt"
run_command "docker info" "$COLLECTOR_OUTPUT_DIR/docker-info.txt"
run_command "docker container ps -all | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"docker container logs \$line\" \"$COLLECTOR_OUTPUT_DIR/docker-container-logs-\$line.txt\"; done" "/dev/null"
run_command "docker container ps -all | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"docker inspect \$line\" \"$COLLECTOR_OUTPUT_DIR/docker-inspect-\$line.txt\"; done" "/dev/null"
run_command "docker network ls | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"docker network inspect \$line\" \"$COLLECTOR_OUTPUT_DIR/docker-network-inspect-\$line.txt\"; done" "/dev/null"
run_command "docker ps | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"docker top \$line\" \"$COLLECTOR_OUTPUT_DIR/docker-top-\$line.txt\"; done" "/dev/null"
run_command "docker version" "$COLLECTOR_OUTPUT_DIR/docker-version.txt"
run_command "podman container ls --all --size" "$COLLECTOR_OUTPUT_DIR/podman-container-ls-all-size.txt"
run_command "podman image ls --all" "$COLLECTOR_OUTPUT_DIR/podman-image-ls-all.txt"
run_command "podman info" "$COLLECTOR_OUTPUT_DIR/podman-info.txt"
run_command "podman container ps -all | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"podman container logs \$line\" \"$COLLECTOR_OUTPUT_DIR/podman-container-logs-\$line.txt\"; done" "/dev/null"
run_command "podman container ps -all | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"podman inspect \$line\" \"$COLLECTOR_OUTPUT_DIR/podman-inspect-\$line.txt\"; done" "/dev/null"
run_command "podman network ls | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"podman network inspect \$line\" \"$COLLECTOR_OUTPUT_DIR/podman-network-inspect-\$line.txt\"; done" "/dev/null"
run_command "podman ps | sed 1d | cut -d\" \" -f 1 | while read line; do run_command \"podman top \$line\" \"$COLLECTOR_OUTPUT_DIR/podman-top-\$line.txt\"; done" "/dev/null"
run_command "podman version" "$COLLECTOR_OUTPUT_DIR/podman-version.txt"
run_command "virsh list --all" "$COLLECTOR_OUTPUT_DIR/virsh-list-all.txt"
run_command "virsh list --name | while read line; do run_command \"virsh domifaddr \$line\" \"$COLLECTOR_OUTPUT_DIR/virsh-domifaddr-\$line.txt\"; done" "/dev/null"
run_command "virsh list --name | while read line; do run_command \"virsh dominfo \$line\" \"$COLLECTOR_OUTPUT_DIR/virsh-dominfo-\$line.txt\"; done" "/dev/null"
Expand Down
116 changes: 116 additions & 0 deletions collectors/files
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# Copyright (C) 2020 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# files collector

if $COLLECTOR_SYSTEM || $COLLECTOR_LOGS || $COLLECTOR_SUSPICIOUS_FILES || $COLLECTOR_USER; then

logging I ">> Searching and collecting files\n"

# exclude paths
run_command "cat \"$CWD/conf/exclude.conf\" | grep -v \"^#\" | grep -v \"^$\" | grep \"^/\" | sed 's:^/:'\"$MOUNT_POINT\"'/:; s://*:/:g; s:/$::g'" "$COLLECTOR_OUTPUT_DIR/.exclude.tmp"
# exclude names
run_command "cat \"$CWD/conf/exclude.conf\" | grep -v \"^#\" | grep -v \"^$\" | grep -v \"^/\"" "$COLLECTOR_OUTPUT_DIR/.exclude.tmp"
run_command "cat \"$CWD/conf/exclude.conf\" | grep -v \"^#\" | grep -v \"^$\" | grep -v \"^/\"" "$COLLECTOR_OUTPUT_DIR/.exclude_names.tmp"
# exclude mounted network file systems
$EXCLUDE_MOUNTED_REMOTE_FILE_SYSTEMS && run_command "get_mount_points_to_exclude" "$COLLECTOR_OUTPUT_DIR/.exclude.tmp"
if [ -s "$COLLECTOR_OUTPUT_DIR/.exclude.tmp" ]; then
logging D "File Systems excluded from collection:\n"
cat "$COLLECTOR_OUTPUT_DIR/.exclude.tmp" >>"$UAC_LOG_FILE"
fi

if $COLLECTOR_SYSTEM || $COLLECTOR_LOGS || $COLLECTOR_SUSPICIOUS_FILES; then

$COLLECTOR_SYSTEM && run_command "cat \"$CWD/conf/system_files.conf\"" >>"$COLLECTOR_OUTPUT_DIR/.merged.tmp"
$COLLECTOR_LOGS && run_command "cat \"$CWD/conf/logs.conf\"" >>"$COLLECTOR_OUTPUT_DIR/.merged.tmp"
$COLLECTOR_SUSPICIOUS_FILES && run_command "cat \"$CWD/conf/suspicious_files.conf\"" >>"$COLLECTOR_OUTPUT_DIR/.merged.tmp"

# first we need to search for directories which names were added to conf files
run_command "ufind \"$MOUNT_POINT\" \"$LOGS_MAX_DEPTH\" \"$COLLECTOR_OUTPUT_DIR/.exclude.tmp\" \"$COLLECTOR_OUTPUT_DIR/.merged.tmp\" \"d\" \"\" " "$COLLECTOR_OUTPUT_DIR/.directories.tmp"
# merge the paths added to conf/logs.conf with the ones found above
run_command "cat \"$COLLECTOR_OUTPUT_DIR/.merged.tmp\" | grep -v \"^#\" | grep -v \"^$\" | grep \"^/\" | sed 's:^/:'\"$MOUNT_POINT\"'/:; s://*:/:g; s:/$::g'" "$COLLECTOR_OUTPUT_DIR/.directories.tmp"
# sort and uniq
run_command "cat \"$COLLECTOR_OUTPUT_DIR/.directories.tmp\" | sort | uniq" "$COLLECTOR_OUTPUT_DIR/.directories_sort_uniq.tmp"
# search paths
run_command "cat \"$COLLECTOR_OUTPUT_DIR/.directories_sort_uniq.tmp\" | while read line; do ufind \"\$line\" \"$LOGS_MAX_DEPTH\" \"$COLLECTOR_OUTPUT_DIR/.exclude_names.tmp\" \"\" \"f\" \"$LOGS_MAX_FILE_SIZE\" \"$DATE_RANGE_T1\" \"$DATE_RANGE_T2\"; done " "$COLLECTOR_OUTPUT_DIR/.files.tmp"
# search for file names
run_command "ufind \"$MOUNT_POINT\" \"$LOGS_MAX_DEPTH\" \"$COLLECTOR_OUTPUT_DIR/.exclude.tmp\" \"$COLLECTOR_OUTPUT_DIR/.merged.tmp\" \"f\" \"$LOGS_MAX_FILE_SIZE\" \"$DATE_RANGE_T1\" \"$DATE_RANGE_T2\"" "$COLLECTOR_OUTPUT_DIR/.files.tmp"

# remove temporary files
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.merged.tmp\"" "/dev/null"
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.directories.tmp\"" "/dev/null"
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.directories_sort_uniq.tmp\"" "/dev/null"

fi

if $COLLECTOR_USER; then

USER_HOME_LIST=`get_user_home_list`

if [ -n "$USER_HOME_LIST" ]; then

echo "$USER_HOME_LIST" | while read USER_HOME_DIR || [ -n "$USER_HOME_DIR" ]; do
USER_HOME_DIR=`sanitize_path "$MOUNT_POINT/$USER_HOME_DIR"`
logging D "Collecting user files from $USER_HOME_DIR\n"

# if home dir is / (like in some systems where root home dir is /)
# uac will limit the search by setting the -maxdepth to 2
FIND_MAXDEPTH_PARAMETER="$USER_FILES_MAX_DEPTH"
if [ "$USER_HOME_DIR" = "/" ]; then
FIND_MAXDEPTH_PARAMETER="2"
fi

# exclude paths
run_command "cat \"$CWD/conf/exclude.conf\" | grep -v \"^#\" | grep -v \"^$\" | grep \"^/\" | sed 's:^/:'\"$USER_HOME_DIR\"'/:; s://*:/:g; s:/$::g'" "$COLLECTOR_OUTPUT_DIR/.exclude.tmp"

# first we need to search for directories which names were added to conf/user_files.conf
run_command "ufind \"$USER_HOME_DIR\" \"$FIND_MAXDEPTH_PARAMETER\" \"$COLLECTOR_OUTPUT_DIR/.exclude.tmp\" \"$CWD/conf/user_files.conf\" \"d\" \"\" " "$COLLECTOR_OUTPUT_DIR/.directories.tmp"
# merge the paths added to conf/user_files.conf with the ones found above
run_command "cat \"$CWD/conf/user_files.conf\" | grep -v \"^#\" | grep -v \"^$\" | grep \"^/\" | sed 's:^/:'\"$USER_HOME_DIR\"'/:; s://*:/:g; s:/$::g'" "$COLLECTOR_OUTPUT_DIR/.directories.tmp"
# sort and uniq
run_command "cat \"$COLLECTOR_OUTPUT_DIR/.directories.tmp\" | sort | uniq" "$COLLECTOR_OUTPUT_DIR/.directories_sort_uniq.tmp"
# search paths
run_command "cat \"$COLLECTOR_OUTPUT_DIR/.directories_sort_uniq.tmp\" | while read line; do ufind \"\$line\" \"$FIND_MAXDEPTH_PARAMETER\" \"$COLLECTOR_OUTPUT_DIR/.exclude.tmp\" \"\" \"f\" \"$USER_FILES_MAX_FILE_SIZE\" \"$DATE_RANGE_T1\" \"$DATE_RANGE_T2\"; done " "$COLLECTOR_OUTPUT_DIR/.files.tmp"
# search for file names
run_command "ufind \"$USER_HOME_DIR\" \"$FIND_MAXDEPTH_PARAMETER\" \"$COLLECTOR_OUTPUT_DIR/.exclude.tmp\" \"$CWD/conf/user_files.conf\" \"f\" \"$USER_FILES_MAX_FILE_SIZE\" \"$DATE_RANGE_T1\" \"$DATE_RANGE_T2\"" "$COLLECTOR_OUTPUT_DIR/.files.tmp"

# remove temporary files
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.directories.tmp\"" "/dev/null"
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.directories_sort_uniq.tmp\"" "/dev/null"

done

fi
fi

# remove UAC working directory from the collection, sort and uniq
run_command "cat \"$COLLECTOR_OUTPUT_DIR/.files.tmp\" | grep -v \"$CWD\" | sort | uniq" "$COLLECTOR_OUTPUT_DIR/files.txt"

if ${TAR_TOOL_AVAILABLE} && ${GZIP_TOOL_AVAILABLE}; then
# archive and compress data
run_command "archive_compress_data \"$COLLECTOR_OUTPUT_DIR/files.txt\" \"$COLLECTOR_OUTPUT_DIR/files.tar.gz\" true"
elif ${TAR_TOOL_AVAILABLE}; then
# archive data
run_command "archive_data \"$COLLECTOR_OUTPUT_DIR/files.txt\" \"$COLLECTOR_OUTPUT_DIR/files.tar\" true"
else
# just copy data
run_command "copy_data \"$COLLECTOR_OUTPUT_DIR/files.txt\" \"$COLLECTOR_OUTPUT_DIR\""
fi

# remove temporary files
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.exclude.tmp\"" "/dev/null"
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.exclude_names.tmp\"" "/dev/null"
run_command "rm -f \"$COLLECTOR_OUTPUT_DIR/.files.tmp\"" "/dev/null"

fi
62 changes: 0 additions & 62 deletions collectors/logs

This file was deleted.

Loading

0 comments on commit 6cddc9a

Please sign in to comment.