ingest_edge_list.cpp File Reference

#include <boost/bind.hpp>
#include <boost/function.hpp>
#include <havoqgt/delegate_partitioned_graph.hpp>
#include <havoqgt/parallel_edge_list_reader.hpp>
#include <havoqgt/environment.hpp>
#include <havoqgt/cache_utilities.hpp>
#include <havoqgt/distributed_db.hpp>
#include <iostream>
#include <assert.h>
#include <deque>
#include <utility>
#include <algorithm>
#include <functional>
#include <fstream>
#include <unistd.h>

Include dependency graph for ingest_edge_list.cpp:

Go to the source code of this file.

Typedefs
typedef havoqgt::distributed_db::segment_manager_type	segment_manager_t
typedef hmpi::delegate_partitioned_graph< segment_manager_t >	graph_type

Functions
void	usage ()
void	parse_cmd_line (int argc, char **argv, std::string &output_filename, uint64_t &delegate_threshold, std::vector< std::string > &input_filenames)
int	main (int argc, char **argv)

Typedef Documentation

typedef hmpi::delegate_partitioned_graph<segment_manager_t> graph_type

Definition at line 78 of file ingest_edge_list.cpp.

typedef havoqgt::distributed_db::segment_manager_type segment_manager_t

Definition at line 77 of file ingest_edge_list.cpp.

Function Documentation

int main	(	int	argc,
		char **	argv
	)

Definition at line 135 of file ingest_edge_list.cpp.

                                {

  int mpi_rank(0), mpi_size(0);

  havoqgt_init(&argc, &argv);
  {
    int mpi_rank = havoqgt_env()->world_comm().rank();
    int mpi_size = havoqgt_env()->world_comm().size();
    havoqgt::get_environment();
    
    if (mpi_rank == 0) {
      std::cout << "MPI initialized with " << mpi_size << " ranks." << std::endl;
      havoqgt::get_environment().print();
    }
    havoqgt_env()->world_comm().barrier();

    std::string                output_filename;
    uint64_t                   delegate_threshold;
    std::vector< std::string > input_filenames;
    
    parse_cmd_line(argc, argv, output_filename, delegate_threshold, input_filenames);

    if (mpi_rank == 0) {
      std::cout << "Ingesting graph from " << input_filenames.size() << " files." << std::endl;
    }

    havoqgt::distributed_db ddb(havoqgt::db_create(), output_filename.c_str());

    segment_manager_t* segment_manager = ddb.get_segment_manager();
    bip::allocator<void, segment_manager_t> alloc_inst(segment_manager);

    //Setup edge list reader
    havoqgt::parallel_edge_list_reader pelr(input_filenames);


    if (mpi_rank == 0) {
      std::cout << "Generating new graph." << std::endl;
    }
    graph_type *graph = segment_manager->construct<graph_type>
        ("graph_obj")
        (alloc_inst, MPI_COMM_WORLD, pelr, pelr.max_vertex_id(), delegate_threshold);


    havoqgt_env()->world_comm().barrier();
    if (mpi_rank == 0) {
      std::cout << "Graph Ready, Calculating Stats. " << std::endl;
    }



    for (int i = 0; i < mpi_size; i++) {
      if (i == mpi_rank) {
        double percent = double(segment_manager->get_free_memory()) /
        double(segment_manager->get_size());
        std::cout << "[" << mpi_rank << "] " << segment_manager->get_free_memory()
                  << "/" << segment_manager->get_size() << " = " << percent << std::endl;
      }
      havoqgt_env()->world_comm().barrier();
    }

    graph->print_graph_statistics();


    //
    // Calculate max degree
    uint64_t max_degree(0);
    for (auto citr = graph->controller_begin(); citr != graph->controller_end(); ++citr) {
      max_degree = std::max(max_degree, graph->degree(*citr));
    }

    uint64_t global_max_degree = havoqgt::mpi::mpi_all_reduce(max_degree, std::greater<uint64_t>(), MPI_COMM_WORLD);

    havoqgt_env()->world_comm().barrier();

    if (mpi_rank == 0) {
      std::cout << "Max Degree = " << global_max_degree << std::endl;
    }

    havoqgt_env()->world_comm().barrier();

  } //END Main MPI
  havoqgt_finalize();
  return 0;
}

Here is the call graph for this function:

void parse_cmd_line	(	int	argc,
		char **	argv,
		std::string &	output_filename,
		uint64_t &	delegate_threshold,
		std::vector< std::string > &	input_filenames
	)

Definition at line 90 of file ingest_edge_list.cpp.

                                                                                                                                              {
  if(havoqgt_env()->world_comm().rank() == 0) {
    std::cout << "CMD line:";
    for (int i=0; i<argc; ++i) {
      std::cout << " " << argv[i];
    }
    std::cout << std::endl;
  }
  
  bool found_output_filename = false;
  delegate_threshold = 1048576;
  input_filenames.clear();
  
  char c;
  bool prn_help = false;
  while ((c = getopt(argc, argv, "o:d:h ")) != -1) {
     switch (c) {
       case 'h':  
         prn_help = true;
         break;
       case 'd':
         delegate_threshold = atoll(optarg);
         break;
      case 'o':
         found_output_filename = true;
         output_filename = optarg;
         break;
      default:
         std::cerr << "Unrecognized option: "<<c<<", ignore."<<std::endl;
         prn_help = true;
         break;
     }
   } 
   if (prn_help || !found_output_filename) {
     usage();
     exit(-1);
   }

   for (int index = optind; index < argc; index++) {
     std::cout << "Input file = " << argv[index] << std::endl;
     input_filenames.push_back(argv[index]);
   }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void usage

(

)

Definition at line 80 of file ingest_edge_list.cpp.

              {
  if(havoqgt_env()->world_comm().rank() == 0) {
    std::cerr << "Usage: -o <string> -d <int> [file ...]\n"
         << " -o <string>   - output graph base filename (required)\n"
         << " -d <int>      - delegate threshold (Default is 1048576)\n"
         << " -h            - print help and exit\n"
         << "[file ...] - list of edge list files to ingest\n\n";
  }
}

Here is the call graph for this function:

Here is the caller graph for this function: