hdfspipe.in

#!/bin/bash

#############################################
#    Copyright (C) 2012 HPCC Systems.
#
#    All rights reserved. This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with All rights reserved. This program is free software: you can redistribute program.  If not, see <http://www.gnu.org/licenses/>.
#############################################

#DO NOT MODIFY THIS FILE!
#CONFIGURATION CHANGES CAN BE DONE HERE: /etc/HPCCSystems/@HDFSCONN_CONF_FILE@

source @HPCC_ETC_DIR@/init.d/hpcc_common
set_environmentvars

source $configs/@HDFSCONN_CONF_FILE@

CLASSPATH=$CLASSPATH:$HADOOP_CONF_DIR

###################################################
# CLASSPATH set up for libhdfs
###################################################
for f in $HADOOP_SHARE_DIR/*.jar ; do
    CLASSPATH=$CLASSPATH:$f
done

for f in $HADOOP_SHARE_DIR/lib/*.jar ; do
    CLASSPATH=$CLASSPATH:$f
done

if [ -n "$TARBALL_HADOOP_LOCATION"  ]; then
    CLASSPATH=$CLASSPATH:$TARBALL_HADOOP_LOCATION/conf

    for f in $TARBALL_HADOOP_LOCATION/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $TARBALL_HADOOP_LOCATION/lib/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done
fi

export CLASSPATH=$CLASSPATH

PID=$$

nodeid=0;
wuid=0;
for ((paramindex=1; paramindex <= $#; paramindex++ ));
 do
   if [ "${!paramindex}" = "-wuid" ];
   then
        ((paramindex++));
        wuid=${!paramindex};
        echo "found wuid: $wuid";
   elif [ "${!paramindex}" = "-nodeid" ];
   then
        ((paramindex++));
        nodeid=${!paramindex};
        echo "found nodeid: $nodeid";
   fi
done;

#the log variable is read from the HPCC Platform config
LOGS_LOCATION=$log
HDFSCONNLOGLOC=$LOGS_LOCATION/mydataconnectors
LOG=$HDFSCONNLOGLOC/@HDFS_CONNECTOR_TYPE@.$nodeid.$PID.$wuid.log
PIPERRLOG=$HDFSCONNLOGLOC/HPCC-FIFO.err.$PID

if [ -e $HDFSCONNLOGLOC ]
  then
    echo "log file found"           >> $LOG
  else
    mkdir $HDFSCONNLOGLOC
fi
username=`id -nu`
echo "Script starting"               >> $LOG
echo "PID: $PID"                     >> $LOG
echo "User: $username"               >> $LOG
echo 'Incoming params: '             >> $LOG
echo $(printf '%q ' ${@})            >> $LOG
echo "NodeId: $nodeid"               >> $LOG
echo "WUID: $wuid"                   >> $LOG

h2hpid=0;
h2hstatus=0;

TARGETCONNECTORNAME="@HDFSCONN_EXE_NAME@";

if [ "$1" = "" ];
then
    echo "Error: No input params detected!" >> $LOG
    exit 1;
elif [ $1 = "-mf" ];
then
    $TARGETCONNECTORNAME ${@}      2>> $LOG;
    h2hstatus=$?;
    h2hpid=$!;
elif [ $1 = "-si" ];
then
    $TARGETCONNECTORNAME  ${@}      2>> $LOG;
    h2hstatus=$?
    h2hpid=$!;
elif [ $1 = "-so" ];
then
    HPCCTMPFILE=/tmp/HPCCTMPFILE-$nodeid-$PID-$wuid;
    if [ -f $HPCCTMPFILE ]
    then
        rm "$HPCCTMPFILE" 			2>> $LOG
        echo "deleted tmp file" 		>> $LOG
    else
        echo "$HPCCTMPFILE does not exist"    >> $LOG
    fi

    echo "redirecting stdin to tmp file ... "   >> $LOG
    cat /dev/stdin > "$HPCCTMPFILE"

	ls -l "$HPCCTMPFILE"                    >> $LOG

    echo "calling hdfsconnector..."         >> $LOG

    $TARGETCONNECTORNAME ${@} -pipepath $HPCCTMPFILE  	2>> $LOG
    h2hpid=$!;
    h2hstatus=$?;
elif [ $1 = "-sop" ];
then
    pipepath=/tmp/HPCCPIPE-$nodeid-$PID-$wuid;
    mkfifo $pipepath 2> $PIPERRLOG;
    chmod 777 $pipepath 2> $PIPERRLOG;

    echo "mkfifo $pipepath setup ..."           >> $LOG

    if test ! -s "$PIPERRLOG"
    then
        rm -f $PIPERRLOG 2> /dev/null
    else
        echo "  WARNING (hdfsconnector mkfifo) error registered in file: $PIPERRLOG " >> $LOG
        exit 1
    fi
    $TARGETCONNECTORNAME  ${@} -pipepath $pipepath	2>> $LOG &

    h2hpid=$!;

    echo h2h pid: ${h2hpid} >> $LOG
    echo "redirecting stdin to named pipe ... "    >> $LOG

    cat < /dev/stdin > "$pipepath" &
    ls -l "$pipepath"                               >> $LOG
    echo "Waiting on hdfsconnector PID ${h2hpid}"   >> $LOG
    wait ${h2hpid};
    h2hstatus=$?;
    echo "Finished waiting on hdfsconnector exit status: ${h2hstatus}" >> $LOG

    if [ ${h2hstatus} -ne 0 ];
    then
        echo "Closing pipe due to hdfsconnector error" >> $LOG
        exec  <&- 2>> $LOG
        exec $pipepath <&- 2>> $LOG
    fi
    rm -f $pipepath 2>> $LOG
else
    echo "Error: check your params."             >> $LOG;
    h2hstatus=1;
fi

echo "$TARGETCONNECTORNAME exited with: ${h2hstatus}"            >> $LOG

exit ${h2hstatus};