9 Leveraging Text Files
9 Leveraging Text Files
9. Leveraging Text Files
1
2
3
4
5
6
# Prints all usernames (the first field) alphabetically
$ cut -d: -f1 /etc/passwd | sort
avahi
backup
daemon
⋮
1
2
3
$ awk -F: '$3>=1000 {print $1}' /etc/passwd \
| xargs -I@ echo 'echo "Hi there, @!" | mail -s greetings @' \
| bash
A First Example: Finding Files
step1:
1
2
$ find $HOME -name animals.txt -print
/home/smith/Work/Writing/Books/Lists/animals.txt
step2:
1
2
3
4
5
6
$ find $HOME -print > $HOME/.ALLFILES
$ head -n3 $HOME/.ALLFILES
/home/smith
/home/smith/Work
/home/smith/Work/resume.pdf
⋮
step3:
1
2
$ grep animals.txt $HOME/.ALLFILES
/home/smith/Work/Writing/Books/Lists/animals.txt
step4: write a one-line script
1
2
3
#!/bin/bash
# $@ means all arguments provided to the script
grep "$@" $HOME/.ALLFILES
1
2
3
4
$ chmod +x ff
$ echo $PATH Check your search path
/home/smith/bin:/usr/local/bin:/usr/bin:/bin
$ mv ff ~/bin
1
2
3
4
5
6
7
8
9
$ ff animal
/home/smith/Work/Writing/Books/Lists/animals.txt
$ ff -i animal | less Case-insensitive grep
/home/smith/Work/Writing/Books/Lists/animals.txt
/home/smith/Vacations/Zoos/Animals/pandas.txt
/home/smith/Vacations/Zoos/Animals/tigers.txt
⋮
$ ff -i animal | wc -l How many matches?
16
Checking Domain Expiration
1
2
3
4
5
6
7
#!/bin/bash
expdate=$(date \
--date $(whois "$1" \
| grep 'Registry Expiry Date:' \
| awk '{print $4}') \
+'%Y-%m-%d')
echo "$expdate $1" # Two values separated by a tab
1
2
$ ./check-expiry example.com
2022-08-13 example.com
1
2
3
4
5
#!/bin/bash
cat domains.txt | while read domain; do
./check-expiry "$domain"
sleep 5 # Be kind to the registrar's server
done
1
$ ./check-expiry-all &> expiry.txt &
1
2
3
4
5
$ cat expiry.txt
2022-08-13 example.com
2022-05-26 oreilly.com
2022-09-17 efficientlinux.com
⋮
1
2
3
# sort the dates and find the next domain to renew
$ sort -n expiry.txt | head -n1
2022-05-26 oreilly.com
1
2
# find domains that have expired or are expiring today
$ awk "\$1<=\"$(date +%Y-%m-%d)\"" expiry.txt
Building an Area Code Database
1
2
3
4
5
6
# areacodes.txt
201 NJ Hackensack, Jersey City
202 DC Washington
203 CT New Haven, Stamford
⋮
989 MI Saginaw
1
2
3
4
5
$ grep -w NJ areacodes.txt
201 NJ Hackensack, Jersey City
551 NJ Hackensack, Jersey City
609 NJ Atlantic City, Trenton, southeast and central west
⋮
1
2
$ grep -w 202 areacodes.txt
202 DC Washington
1
2
3
4
5
$ grep Washing areacodes.txt
202 DC Washington
227 MD Silver Spring, Washington suburbs, Frederick
240 MD Silver Spring, Washington suburbs, Frederick
⋮
1
2
$ wc -l areacodes.txt
375 areacodes.txt
1
2
3
# Find the state with the most area codes (the winner is California with 38):
$ cut -f2 areacodes.txt | sort | uniq -c | sort -nr | head -n1
38 CA
1
2
3
4
5
6
7
# Convert the file to CSV format
$ awk -F'\t' '{printf "%s,%s,\"%s\"\n", $1, $2, $3}' areacodes.txt \
> areacodes.csv
$ head -n3 areacodes.csv
201,NJ,"Hackensack, Jersey City"
202,DC,"Washington"
203,CT,"New Haven, Stamford"
1
2
3
# Collate all area codes for a given state onto a single line
$ awk '$2~/^NJ$/{ac=ac FS $1} END {print "NJ:" ac}' areacodes.txt
NJ: 201 551 609 732 848 856 862 908 973
1
2
3
4
5
6
7
8
9
# collate for each state, using arrays and for loops
$ awk '{arr[$2]=arr[$2] " " $1} \
END {for (i in arr) print i ":" arr[i]}' areacodes.txt \
| sort
AB: 403 780
AK: 907
AL: 205 251 256 334 659
⋮
WY: 307
1
2
3
4
5
6
7
8
# A simple example is the areacode script
#!/bin/bash
if [ -n "$1" ]; then
grep -iw "$1" areacodes.txt
fi
$ areacode 617
617 MA Boston
Building a Password Manager
1
2
3
4
5
6
7
8
9
$ touch vault Create an empty file
$ chmod 600 vault Set file permissions
$ emacs vault Edit the file
$ cat vault
sally fake1 google.com account
ssmith fake2 dropbox.com account for work
s999 fake3 Bank of America account, bankofamerica.com
smith2 fake4 My blog at wordpress.org
birdy fake5 dropbox.com account for home
1
2
$ mkdir ~/etc
$ mv vault ~/etc
1
2
3
4
5
6
7
8
$ cd ~/etc
$ grep sally vault Match a username
sally fake1 google.com account
$ grep work vault Match the notes
ssmith fake2 dropbox.com account for work
$ grep drop vault Match multiple lines
ssmith fake2 dropbox.com account for work
birdy fake5 dropbox.com account for home
1
2
3
4
# pman version 1
#!/bin/bash
# Just print matching lines
grep "$1" $HOME/etc/vault
1
2
$ chmod 700 pman
$ mv pman ~/bin
1
2
3
4
5
6
7
8
$ pman goog
sally fake1 google.com account
$ pman account
sally fake1 google.com account
ssmith fake2 dropbox.com account for work
s999 fake3 Bank of America account, bankofamerica.com
birdy fake5 dropbox.com account for home
$ pman facebook (produces no output)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# pman version 2
#!/bin/bash
# Capture the script name.
# $0 is the path to the script, and basename prints the final filename.
PROGRAM=$(basename $0)
# Location of the password vault
DATABASE=$HOME/etc/vault
# Ensure that at least one argument was provided to the script.
# The expression >&2 directs echo to print on stderr instead of stdout.
if [ $# -ne 1 ]; then
>&2 echo "$PROGRAM: look up passwords by string"
>&2 echo "Usage: $PROGRAM string"
exit 1
fi
# Store the first argument in a friendly, named variable
searchstring="$1"
# Search the vault and print an error message if nothing matches
grep "$searchstring" "$DATABASE"
if [ $? -ne 0 ]; then
>&2 echo "$PROGRAM: no matches for '$searchstring'"
exit 1
fi
1
2
3
4
5
6
7
8
$ pman
pman: look up passwords by string
Usage: pman string
$ pman smith
ssmith fake2 dropbox.com account for work
smith2 fake4 My blog at wordpress.org
$ pman xyzzy
pman: no matches for 'xyzzy'
1
2
3
4
5
sally fake1 google google.com account
ssmith fake2 dropbox dropbox.com account for work
s999 fake3 bank Bank of America account, bankofamerica.com
smith2 fake4 blog My blog at wordpress.org
birdy fake5 dropbox2 dropbox.com account for home
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#!/bin/bash
PROGRAM=$(basename $0)
DATABASE=$HOME/etc/vault
if [ $# -ne 1 ]; then
>&2 echo "$PROGRAM: look up passwords"
>&2 echo "Usage: $PROGRAM string"
exit 1
fi
searchstring="$1"
# Look for exact matches in the third column
match=$(awk '$3~/^'$searchstring'$/' "$DATABASE")
# If the search string doesn't match a key, find all matches
if [ -z "$match" ]; then
match=$(awk "/$searchstring/" "$DATABASE")
fi
# If still no match, print an error message and exit
if [ -z "$match" ]; then
>&2 echo "$PROGRAM: no matches for '$searchstring'"
exit 1
fi
# Print the match
echo "$match"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#!/bin/bash
PROGRAM=$(basename $0)
DATABASE=$HOME/etc/vault
if [ $# -ne 1 ]; then
>&2 echo "$PROGRAM: look up passwords"
>&2 echo "Usage: $PROGRAM string"
exit 1
fi
searchstring="$1"
# Look for exact matches in the third column
match=$(awk '$3~/^'$searchstring'$/' "$DATABASE")
# If the search string doesn't match a key, find all matches
if [ -z "$match" ]; then
match=$(awk "/$searchstring/" "$DATABASE")
fi
# If still no match, print an error message and exit
if [ -z "$match" ]; then
>&2 echo "$PROGRAM: no matches for '$searchstring'"
exit 1
fi
# Print the match
echo "$match"
1
$ gpg --quick-generate-key your_email_address default default never
1
2
3
4
5
6
7
8
9
10
$ cd ~/etc
$ gpg -e -r your_email_address vault
$ ls vault*
vault vault.gpg
$ gpg -d -q vault.gpg
Passphrase: xxxxxxxx
sally fake1 google google.com account
ssmith fake2 dropbox dropbox.com account for work
⋮
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
# pman version 4
#!/bin/bash
PROGRAM=$(basename $0)
# Use the encrypted file
DATABASE=$HOME/etc/vault.gpg
if [ $# -ne 1 ]; then
>&2 echo "$PROGRAM: look up passwords"
>&2 echo "Usage: $PROGRAM string"
exit 1
fi
searchstring="$1"
# Store the decrypted text in a variable
decrypted=$(gpg -d -q "$DATABASE")
# Look for exact matches in the third column
match=$(echo "$decrypted" | awk '$3~/^'$searchstring'$/')
# If the search string doesn't match a key, find all matches
if [ -z "$match" ]; then
match=$(echo "$decrypted" | awk "/$searchstring/")
fi
# If still no match, print an error message and exit
if [ -z "$match" ]; then
>&2 echo "$PROGRAM: no matches for '$searchstring'"
exit 1
fi
# Print the match
echo "$match"
1
2
3
4
5
6
7
$ pman dropbox
Passphrase: xxxxxxxx
ssmith fake2 dropbox dropbox.com account for work
$ pman drop
Passphrase: xxxxxxxx
ssmith fake2 dropbox dropbox.com account for work
birdy fake5 dropbox2 dropbox.com account for home
1
2
3
# Support comments in the password vault—lines that begin with a pound sign (#)—so you can make notes about the entries. To do so, update the script to pipe the decrypted contents to grep -v to filter any lines that begin with a pound sign:
decrypted=$(gpg -d -q "$DATABASE" | grep -v '^#')
1
2
3
4
5
6
$ cd ~/etc
$ gpg vault.gpg Decrypt
Passphrase: xxxxxxxx
$ emacs vault Use your favorite text editor
$ gpg -e -r your_email_address vault Encrypt for yourself
$ rm vault
1
export GPG_TTY=$(tty)
Editing Encrypted Files Directly
1
2
3
4
5
# For emacs, set up the EasyPG package, which is built-in. Add the following lines to the configuration file $HOME/.emacs and restart emacs. Replace the string GnuPG ID here with the email address associated with your key, such as smith@example.com:
(load-library "pinentry")
(setq epa-pinentry-mode 'loopback)
(setq epa-file-encrypt-to "GnuPG ID here")
(pinentry-start)
1
2
3
# For vim, try the plugin vim-gnupg and add these lines to the configuration file $HOME/.vimrc:
let g:GPGPreferArmor=1
let g:GPGDefaultRecipients=["GnuPG ID here"]
1
2
# Consider creating an alias to edit the password vault conveniently
$ alias pwedit="$EDITOR $HOME/etc/vault.gpg"
This post is licensed under CC BY 4.0 by the author.