test/parallel-export.cpp

#include "DAPTA.hpp"

#include <set>
#include <fstream>
#include <iostream>
#include <iomanip>
#undef SEEK_SET
#undef SEEK_CUR
#undef SEEK_END
#include "mpi.h"
using namespace DAPTA;

#define LENGTH_EXPORT_FILE 0
#define STRING_EXPORT_FILE 1

typedef Mesh<Tri3MeshTraits> TriMesh;
TriMesh mesh;
RCBDecomposer<TriMesh::VertexType> rcb;
std::vector<TriMesh::VertexHandle> points;

int main(int argc, char **argv) {
   int          myrank, nprocs;
   MPI_Status   status;
   
   // Initialize MPI
   MPI_Init(&argc, &argv);

   // Get my rank id and number of MPI processes
   MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
   MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
   
   if(myrank == 0) {
      // Load in the mesh
      mesh.ClearMesh();
      std::ifstream meshfile;
      int num;
      
      meshfile.open(argv[1]);
      if(!meshfile) {
         std::cout << "ERROR READING FILE!" << std::endl;
         exit(1);
      }
      
      mesh.ReadFile(meshfile);
      meshfile.close();
      points = mesh.GetCentres();
      
      rcb.loadVertices(points);
      
      for(int i=1; i<nprocs; i++) {
         //std::cout << "client(" << myrank << "): GETTING DOMAIN FOR DOMAIN: " << i << "/" << nprocs << std::endl;
         
         std::vector<TriMesh::VertexHandle> myNodes = rcb.decompose(nprocs, i);
         std::set<TriMesh::VertexHandle> part_vert;
         std::set<TriMesh::ElementHandle> part_elem;
         std::string export_file;
         
         for(int j=0; j<myNodes.size(); j++) {
            //std::cout << "   client(" << myrank << "): DOING ELEMENT NUMBER: " << myNodes.at(j)->globalID << std::endl;
            
            TriMesh::ElementHandle this_elem = mesh.FindElementFromIndex(myNodes.at(j)->globalID);
            part_elem.insert(this_elem);
            for(int k=0; k<TriMesh::ElementType::num_vertices; k++) {
               part_vert.insert(this_elem->vertices.at(k));
            }
         }
         
         //std::cout << "client(" << myrank << "): GETTING EXPORT FILE FOR DOMAIN: " << i << "/" << nprocs << std::endl;         
         export_file = mesh.ExportDomainFile(part_vert, part_elem);

         const char* char_send = export_file.c_str();
         //strcpy(char_send, export_file.c_str());
         int length_export_file = strlen(char_send);
         
         //std::cout << "client(" << myrank << "): SENDING LENGTH OF EXPORT FILE FOR DOMAIN: " << i << "/" << nprocs << std::endl;
         //std::cout << "   client(" << myrank << "): LENGTH IS: " << length_export_file << std::endl;
         MPI_Send(&length_export_file, 1, MPI_INT, i, LENGTH_EXPORT_FILE, MPI_COMM_WORLD);

         //std::cout << "client(" << myrank << "): SENDING EXPORT FILE FOR DOMAIN: " << i << "/" << nprocs << std::endl;
         MPI_Send((void *)char_send, length_export_file, MPI_CHAR, i, STRING_EXPORT_FILE, MPI_COMM_WORLD);
         
         //std::cout << "client(" << myrank << "): DONE SENDING TO DOMAIN: " << i << "/" << nprocs << std::endl;
      }
      
      //std::cout << "client(" << myrank << "): NOW LOADING IN THE MESH FILE..." << std::endl;
      std::vector<TriMesh::VertexHandle> myNodes = rcb.decompose(nprocs, 0);
      myNodes = rcb.decompose(nprocs, 0);
      std::set<TriMesh::VertexHandle> part_vert;
      std::set<TriMesh::ElementHandle> part_elem;
      std::string export_file;
      
      for(int j=0; j<myNodes.size(); j++) {
         TriMesh::ElementHandle this_elem = mesh.FindElementFromIndex(myNodes.at(j)->globalID);
         part_elem.insert(this_elem);
         for(int k=0; k<TriMesh::ElementType::num_vertices; k++) {
            part_vert.insert(this_elem->vertices.at(k));
         }
      }
      
      //std::cout << "client(" << myrank << "): READING MESH..." << std::endl;
      export_file = mesh.ExportDomainFile(part_vert, part_elem);
      std::stringstream export_file_stream;
      export_file_stream.str(export_file);
      
      mesh.ClearMesh();
      mesh.ReadFile(export_file_stream);
   }
   else {
      //std::cout << "client(" << myrank << "): RECEIVING DATA..." << std::endl;
      int  length_export_file;
      char export_file_char[] = "";
      //std::cout << "client(" << myrank << "): RECEIVING LENGTH" << std::endl;
      MPI_Recv(&length_export_file,                1,  MPI_INT, 0, LENGTH_EXPORT_FILE, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
      //std::cout << "   client(" << myrank << "): RECEIVED LENGTH: " << length_export_file << std::endl;
      //std::cout << "client(" << myrank << "): RECEIVING EXPORT FILE" << std::endl;
      MPI_Recv(export_file_char, length_export_file, MPI_CHAR, 0, STRING_EXPORT_FILE, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
      //std::cout << "client(" << myrank << "): RECEIVED EXPORT FILE" << std::endl;
      
      //std::cout << "client(" << myrank << "): LOADING THE MESH..." << std::endl;
      std::string       export_file(export_file_char);
      std::stringstream export_file_stream;
      export_file_stream.str(export_file);
      
      //std::cout << "client(" << myrank << "): CLEARING MESH..." << std::endl;
      mesh.ClearMesh();
      //std::cout << "client(" << myrank << "): READING MESH..." << std::endl;
      mesh.ReadFile(export_file_stream);
   }
   
   std::cout << "client(" << myrank << "): ================================================" << std::endl;
   for(TriMesh::VertexMapIterator v_iter=mesh.VertexMapBegin(); v_iter!=mesh.VertexMapEnd(); ++v_iter) {
      std::cout << std::setprecision(4);
      std::cout << "client(" << myrank << "): " << (*v_iter)->globalID << "\t";
      for(int i=0; i<TriMesh::VertexType::dimension; i++) {
         std::cout << (*v_iter)->coords.at(i) << "\t";
      }
      std::cout << (*v_iter) << std::endl;
   }
   std::cout << "client(" << myrank << "): " << std::endl;
   for(TriMesh::ElementMapIterator e_iter=mesh.ElementMapBegin(); e_iter!=mesh.ElementMapEnd(); ++e_iter) {
      std::cout << "client(" << myrank << "): " << (*e_iter)->globalID << "\t";
      for(int i=0; i<TriMesh::ElementType::num_vertices; i++) {
         std::cout << (*e_iter)->vertices.at(i)->globalID << "\t";
      }
      std::cout << (*e_iter) << std::endl;
   }
   std::cout << "client(" << myrank << "): ================================================" << std::endl;
   
   //std::cout << "client(" << myrank << "): WAITING FOR ALL PROCESSES..." << std::endl;
   MPI_Barrier(MPI_COMM_WORLD);
   
   // Shut Down MPI
   MPI_Finalize();
   
   return 0;
}

---