Associative arrays (GNU Parallel)

Uit De Vliegende Brigade
Naar navigatie springen Naar zoeken springen

How to use elements from associative arrays as arguments for Parallel?

No export

You can't export associative arrays and there was also an issue with subshells - Or maybe that's the same problem.

Examples

In the file below, there are 5 direct approaches, none of which works, and 2 indirect approaches, that both work:

#!/bin/bash
#
#
################################################################################
# Create an array
################################################################################
#
unset j
declare -Ag j

j[1,c1]="cell1,c1"
j[1,c2]="cell1,c2"
j[1,c3]="cell1,c3"
j[2,c1]="cell2,c1"
j[2,c2]="cell2,c2"
j[2,c3]="cell2,c3"
j[3,c1]="cell3,c1"
j[3,c2]="cell3,c2"
j[3,c3]="cell3,c3"
j[4,c1]="cell4,c1"
j[4,c2]="cell4,c2"
j[4,c3]="cell4,c3"
j[5,c1]="cell5,c1"
j[5,c2]="cell5,c2"
j[5,c3]="cell5,c3"

j_rows=5


################################################################################
# test_function
################################################################################
#
test_function()
{
	echo "test_function()"
	echo "	argument 1: $1"
	echo "	argument 2: $2"
	echo "	argument 3: $3"
}

export -f test_function


################################################################################
# Parallelise this
################################################################################
#
#
# Just 3 times the counter - Works fine
########################################
#
# seq $j_rows | parallel test_function {} {} {}


# Some direct approaches that don't work
########################################
#
# seq $j_rows | parallel test_function {} ${j[{},c1]} ${j[{},c2]}
# seq $j_rows | parallel 'test_function {} ${j[{},c1]} ${j[{},c2]}'
# seq $j_rows | parallel test_function {} "${j[{},c1]}" "${j[{},c2]}"
# seq $j_rows | parallel test_function {} "${j['{}',c1]}" "${j['{}',c2]}"
# parallel 'test_function {} "${j['{}',c1]}" "${j['{}',c2]}"' ::: $(seq $j_rows)


# OK - With intermediate tmp file
########################################
#
# * This is an indirect approach: Store the associative array in a data
#   structure saved in /tmp
# * Impressively simple code
#
#
for i in $(seq $j_rows); do
    echo "$i|${j[$i,c1]}|${j[$i,c2]}|${j[$i,c3]}"
done > /tmp/data.txt
parallel --colsep '\|' test_function :::: /tmp/data.txt


# OK - With intermediate variable
########################################
#
# * Rather than storing the associative array on storage, just use a variable
# * This should be a fraction faster
#
data=""
for i in $(seq $j_rows); do
    line="$i|${j[$i,c1]}|${j[$i,c2]}|${j[$i,c3]}"
    # Append the line to data with a newline, except on the last iteration
    [ -z "$data" ] && data="$line" || data="$data"$'\n'"$line"
done
parallel --colsep '\|' test_function :::: <(echo -e "$data")