function [data, header] = load_data(fname, varargin)
% load_data  Loads columns of data from ASCII text files. Figures out
%            column data types, column names and header comments that start
%            with comment character #.
%
%
% Synposis:
%   [data, header] = load_data(filename, ...)
%
% Input:
%   filename            = input filename of data text file.
%
%
%   Optional parameters follow the form '-option'[, argument].
%        '-type' :  Format of data return value.
%                   Possible Values: 'struct', 'cell' or 'matrix'.
%                  'struct': Returns data columns as named members of a struct.
%                             Default behavior when column names are found in file.
%
%                   'cell' : Returns data as a cell array of unnamed columns.
%                            Default behavior when column names are NOT found in file.
%
%                  'matrix': Returns data as numeric matrix. Only works if columns
%                            are all numeric type.
%                            Default behavior when column names are NOT found in
%                            file AND all columns are numeric.
%
%
%       '-delimiter': List of delimiters as a string.
%                     Default: ' \t' (whitespace)
%
%       '-multipledelimsasone': true or false
%                         true: Treats multiple delimiters as a single delimiter.
%                         false: replace missing data by empty value string
%                         Default: true for whitespace delimiter, false for others.
%                         If true and row data is inappropriate, error occurs.
%
%       '-emptyvaluechar' : string to replace empty character type  values.
%                             Default ''
%
%       '-emptyvaluenumeric' : numeric value to replace empty numeric values.
%                             Default NaN
%
%
%       '-commentprefix' : character that identifies a comment line in
%                          header. Default '#'
%
%
% Output:
%   data is a structure or cell array and header is a cell array of
%   strings.

global type
global delimiter
global multipledelimsasone
global emptyvaluechar
global emptyvaluenumeric
global commentprefix

if nargin < 1
    error ('load_data:wrongargs', 'Need at least one parameter.');
end


% default data output type
type = 'struct';

% default
delimiter = ' \t';
emptyvaluechar = '';
emptyvaluenumeric = NaN;
commentprefix = '#';
multipledelimsasone_set = false;
only_spec_cols = false;

%parse_arguments(varargin);


% parse the options
n_varargs = length (varargin);
expect_arg = false;
for i = 1:n_varargs
    arg = varargin{i};
    if ~expect_arg && ischar (arg) && arg (1) == '-'
        switch arg
            case '-type',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    type = varargin{i + 1};
                    assert( strcmpi(type,'struct') || strcmpi(type,'cell') || strcmpi(type,'matrix'),'Invalid value for -type option.')
                else
                    error ('load_data:noarg', 'Required argument for -type missing.');
                end
            case '-format',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    fmt_in = varargin{i + 1};
                else
                    error ('load_data:noarg', 'Required argument for -format missing.');
                end
                
            case '-delimiter',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    delimiter = varargin{i + 1};
                else
                    error ('load_data:noarg', 'Required argument for -delimiter missing.');
                end
            case '-multipledelimsasone',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    multipledelimsasone = varargin{i + 1};
                    multipledelimsasone_set = true;
                    assert( islogical(multipledelimsasone) ,'Invalid value for -multipledelimsasone option. Use true or false.')
                else
                    error ('load_data:noarg', 'Required argument for -multipledelimsasone missing.');
                end
            case '-columns',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    columns = varargin{i + 1};
                    only_spec_cols = true;
                    assert( isnumeric(columns) && ~isempty(columns) ,'Invalid value for -multipledelimsasone option. Use true or false.')
                else
                    error ('load_data:noarg', 'Required argument for -multipledelimsasone missing.');
                end
                
                
            case '-emptyvaluechar',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    emptyvaluechar = varargin{i + 1};
                else
                    error ('load_data:noarg', 'Required argument for -emptyvaluechar missing.');
                end
            case '-emptyvaluenumeric',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    emptyvaluenumeric = varargin{i + 1};
                else
                    error ('load_data:noarg', 'Required argument for -emptyvaluenumeric missing.');
                end
            case '-commentprefix',
                if i + 1 <= n_varargs
                    expect_arg = true;
                    commentprefix = varargin{i + 1};
                else
                    error ('load_data:noarg', 'Required argument for -commentprefix missing.');
                end
            otherwise, error ('load_data:unknownarg', 'Unknown argument %q.', arg);
        end
    else
        if ~expect_arg
            error ('load_data:unexparg', 'Unexpected argument at position %d in varargin.', i);
        end
        expect_arg = false;
    end
end






delimiter_is_whitespace = strcmp(strtrim(delimiter),'\t');

if ~multipledelimsasone_set
    if delimiter_is_whitespace
        multipledelimsasone = true;
    else
        multipledelimsasone = false;
    end
end

fid = fopen(fname);
if fid==-1
    error('File not found or permission denied');
end

% Read header lines that begin with #.
l_h = 1;
in_header = 1;
comments = {};
while in_header==1
    end_of_header = ftell(fid);
    line = fgetl(fid);
    t_line = strtrim(line);
    if t_line(1)==commentprefix
        comments{l_h} = t_line(2:end);
        l_h = l_h + 1;
    else
        in_header = 0;
    end
end
comments = comments';
end_of_line_one = ftell(fid);

header.comments = comments;


if ~exist('fmt_in','var')
    fmt_in='-';
end


[fmts, first_row, first_is_fieldnames ] = grok_data(fid, line, fmt_in);
if first_is_fieldnames
    c_names = first_row;
    header.field_names = c_names';
end


n_cols = length(fmts);
successfully_read_whole_file = false;

if only_spec_cols
    n_cols = length(columns);
else
    columns = 1:n_cols;
end

[~,N_lines] = system(['wc -l ' fname]);
N_lines = str2double(strtok(N_lines));

N_lines
%columns


C = cell(n_cols,1);


for c_i=1:n_cols
    if strcmp(fmts{columns(c_i)},'%q')
        C{c_i} = cell(N_lines,1);
    elseif strcmp(fmts{columns(c_i)},'%f')
        C{c_i} = zeros(N_lines,1);
    end
end

attempt = 0;
while ~successfully_read_whole_file
    
    fmt_string = strtrim(strcat(fmts{:}));
    % Read all of data from beginning
    if first_is_fieldnames
        fseek(fid,end_of_line_one,'bof');
    else
        fseek(fid,end_of_header,'bof');
    end
    
    if ~only_spec_cols
        C = textscan(fid, fmt_string,'Delimiter',delimiter,'MultipleDelimsAsOne',multipledelimsasone,'EmptyValue',emptyvaluenumeric,'TreatAsEmpty',{'NA','na','N/A'});
    else
        stopped = false;
        line_no = 0;
        while ~stopped
            C_full = textscan(fid, fmt_string,1,'Delimiter',delimiter,'MultipleDelimsAsOne',multipledelimsasone,'EmptyValue',emptyvaluenumeric,'TreatAsEmpty',{'NA','na','N/A'});

            if all(cellfun(@isempty,C_full))
                stopped = true;
            end
            
            if ~stopped
                line_no = line_no +1;
                for c_i=1:n_cols
                    C{c_i}(line_no) = C_full{columns(c_i)};
                end
                
            end
            
            if feof(fid)
                stopped = true;
            end
            
        end
        
        if line_no < N_lines
            for c_i=1:n_cols
                C{c_i}(line_no +1:end) = [];
            end
        end
        
    end
    
    fmt_string
    %C
    % Check if end of file was reached
    if ~feof(fid)
        attempt = attempt + 1;
        disp(['Did not reach end of file! Trying again. Attempt : ' num2str(attempt)])
        % Since eof wasn't reached, find column that caused trouble
        col_length = zeros(n_cols,1);
        for c=1:n_cols
            col_length(c) = size(C{c},1);
        end
        c_fail = find(col_length~=col_length(1));
        c_fail_should_be = [c_fail(1) : n_cols]';
        assert(all(c_fail==c_fail_should_be),'End of file was not reached in a weird way! Quitting.')        
        % Set that column to character and reload data
        fmts{c_fail(1)}='%q';
    else
        successfully_read_whole_file = true;
    end
    
end

fclose(fid);


% Replace empty character data by specified value.
if ~isempty(emptyvaluechar)
    for c_i=1:n_cols
        c = columns(c_i);
        if strcmp(fmts{c},'%q')
            C{c_i}(cellfun(@isempty,C{c_i})) = {emptyvaluechar};
        end
    end
end

%% Recast Data as requested: struct, cell or matrix.
if first_is_fieldnames && strcmpi(type,'struct')
    for c_i=1:n_cols
        c = columns(c_i);
        if strcmp(fmts{c},'%q')
            eval(['data.' c_names{c} ' = C{c_i} ;' ]);
        elseif strcmp(fmts{c},'%f')
            eval(['data.' c_names{c} ' = C{c_i};' ]);
        end
    end
else
    data = C;
end

if strcmpi(type,'matrix')
    num_cols = strcmp(fmts(columns),'%f');
    char_cols = strcmp(fmts(columns),'%q');
    M = cell2mat(data(num_cols));
    data = [data(char_cols), M];
end


end


function [fmts, first_row, first_is_fieldnames] = grok_data(fid, first_line, fmt_in )


global delimiter
global multipledelimsasone


delimiter_is_whitespace = strcmp(strtrim(delimiter),'\t');
% Read possible line of field names (or first line of data)
if multipledelimsasone || ~delimiter_is_whitespace
    first_line = strtrim(first_line);
end


first_row = textscan(first_line,'%q','Delimiter',delimiter,'MultipleDelimsAsOne',multipledelimsasone);
first_row = first_row{1};
n_cols = length(first_row);

first_is_fieldnames = true;
for c=1:n_cols
    if length(first_row{c})<50 && ~isnan(str2double(first_row{c}))
        first_is_fieldnames = false;
    end
end


fmts = cell(n_cols,1);
fmts(:) = {fmt_in};

need_another_row = true;
end_of_file_reached = false;

while need_another_row && ~end_of_file_reached
    line = fgetl(fid);
    
    if ~ischar(line)
        print 'Too much missing data. Assuming char data type for columns'
        end_of_file_reached = true;
        fmts(strcmp(fmts,'-'))={'%q'};
    else
        if multipledelimsasone || ~delimiter_is_whitespace
            line = strtrim(line);
        end
        
        next_row = textscan(line, '%q','Delimiter',delimiter,'MultipleDelimsAsOne',multipledelimsasone);
        next_row = next_row{1};
        
        assert(n_cols>=length(next_row),'Too many fields in row of data.')
        if n_cols>length(next_row)
            next_row(end+1:n_cols) = {''};
        end
        
        for c=1:n_cols
            if ~isempty(next_row{c}) && ~strcmp(fmts{c},'%q')
                if length(next_row{c})>25 || isnan(str2double_treat_commas(next_row{c}))
                    fmts{c} = '%q';
                elseif strcmp(fmts{c},'-')
                    fmts{c} = '%f';
                end
            end
        end
        need_another_row = ~all(cellfun(@isempty, strfind(fmts,'-')));
    end
end


end

