Click here to Skip to main content
15,887,214 members
Please Sign up or sign in to vote.
0.00/5 (No votes)
See more:
Hi, i have a reqiurement to load bulk of csv files to sql table. some times, some columns could not come in csv file(some times 100 columns and some times 80 cloumns).That time the package is getting failed. can any one suggest how to create a table dynamically based on csv file structure.
Posted
Comments
CHill60 11-Jun-15 8:08am    
What is the error? Is it because the table already exists? Try dropping the table first (if it exists)
See also Handling BULK Data insert from CSV to SQL Server[^]
Member 11494685 11-Jun-15 9:23am    
if any column is not present in source(csv) file then package will fails.

1 solution

First you have to process each row by row data and check your data is proper , based on that , you have to sort out defected data and correct data separated ,

and then show them in two differnt grids , one with correct data , and one with not corrected data ,

Correct data from the grid which gets carried forward to update into database ,

and not corrected data shown in other grid give the option to extract that into other csv , so he will correct that data and upload those employee data again

below check my code

protected void UploadButton_Click(object sender, EventArgs e)
        {
            userlist = userlist + ApConfig.CurrentCompany.CompId.ToString() + "/";
            divRecNotDefectedNotFound.Visible = false;
            panHeaderNotDefected.Visible = false;

            divRecDefectedNotFound.Visible = false;
            panHeaderDefected.Visible = false;

            grdDefectRecord.DataSource = null;
            grdDefectRecord.DataBind();
            ViewState["Defect"] = null;
            divExportToCSVDefetedRecords.Visible = false;

            divExToCSVUploadedRecords.Visible = false;
            divButtonProceed.Visible = false;
            grdNotDefect.DataSource = null;
            grdNotDefect.DataBind();
            ViewState["NotDefect"] = null;

            DataTable myDBTable = new DataTable();
            DataTable myManagersTable = new DataTable();

            objemp = new Employee();
            int compId = Convert.ToInt32(ApConfig.CurrentCompany.CompId);
            try
            {
                if (FileUploader.HasFile)
                {
                    if (DataHelper.chkCsvFileType(FileUploader.FileName))
                    {
                        filepath = userlist + FileUploader.FileName;
                        string extension = Path.GetExtension(FileUploader.PostedFile.FileName);

                        if (!Directory.Exists(Server.MapPath(userlist)))
                        {
                            Directory.CreateDirectory(Server.MapPath(userlist));
                        }

                        if (File.Exists(Server.MapPath(filepath)))
                        {
                            File.Delete(Server.MapPath(filepath));
                        }
                        FileUploader.SaveAs(Server.MapPath(filepath));

                        DataTable myExcelTable = new DataTable();

                        myExcelTable.Columns.Add("FirstName", typeof(string));
                        myExcelTable.Columns.Add("LastName", typeof(string));
                        myExcelTable.Columns.Add("Title", typeof(string));
                        // Read the file and display it line by line.

                        string line;
                        using (System.IO.StreamReader file = new System.IO.StreamReader(Server.MapPath(filepath)))
                        {
                            while ((line = file.ReadLine()) != null)
                            {
                                string[] rec = line.Split(',');
                                rec = rec.Select(x => x.Replace("|", ",")).ToArray();
                                if (rec.Length == 55)
                                {
                                    DataRow dr = myExcelTable.NewRow();
                                    if (rec[0] != null)
                                    {
                                        dr["FirstName"] = rec[0];
                                    }
                                    else
                                    {
                                        dr["FirstName"] = "";
                                    }
                                    if (rec[1] != null)
                                    {
                                        dr["LastName"] = rec[1];
                                    }
                                    else
                                    {
                                        dr["LastName"] = "";
                                    }
                                    if (rec[2] != null)
                                    {
                                        dr["Title"] = rec[2];
                                    }
                                    else
                                    {
                                        dr["Title"] = "";
                                    }
                                    
                                    myExcelTable.Rows.Add(dr);
                                }
                                else
                                {
                                    lblMessage.Visible = true;
                                    lblMessage.Text = "Columns are not in specified format.";
                                    lblMessage.ForeColor = System.Drawing.Color.Red;
                                    return;
                                }
                            }
                        }

                        if (myExcelTable == null)
                        {
                            lblMessage.Text = "File is empty.";
                            return;
                        }

                        // Get Managers List
                        List<ActusLibrary.Employee> managerList = objemp.GetCompanyManagerList(compId);
                        myManagersTable = DataHelper.LinqToDataTable(managerList);

                        //TO CHECK ALL REQUIRED HEADER COLUMN EXISTS OR NOT
                        if (myExcelTable.Columns.Count == 2)
                        {
                            if (
                                myExcelTable.Rows[0][0].ToString() == "FirstName" &&
                                myExcelTable.Rows[0][1].ToString() == "LastName" &&
                                myExcelTable.Rows[0][2].ToString() == "Title" 

                                )
                            {
                                int licenceCount = ActusLibrary.Employee.GetLicenseCount(compId);
                                hdnIsLicenceExceeding.Value = licenceCount.ToString();
                                if (licenceCount == 0)
                                {
                                    lblMessage.Visible = true;
                                    //lblMessage.Text = " You have used your all Licences. To Add more employee please contact to Administrator. Upload failed.";
                                    lblMessage.ForeColor = System.Drawing.Color.Red;
                                    //return;
                                }
                                if (myExcelTable.Rows.Count > 0)
                                {
                                    if (myExcelTable.Rows.Count - 1 > licenceCount)
                                    {
                                        //lblMessage.Visible = true;
                                        //lblMessage.Text = " You can add only  " + licenceCount + " licences. Upload failed.";
                                        //lblMessage.ForeColor = System.Drawing.Color.Red;
                                        //return;
                                    }
                                }
                                else
                                {
                                    lblMessage.Visible = true;
                                    lblMessage.Text = "File is empty.";
                                    lblMessage.ForeColor = System.Drawing.Color.Red;
                                    return;
                                }

                                //CREATE DEFECT DATA TABLE

                                DataTable dtDefect = new DataTable();

                                dtDefect.Columns.Add("FirstName", typeof(string));
                                dtDefect.Columns.Add("LastName", typeof(string));
                                dtDefect.Columns.Add("Title", typeof(string));


                                //CREATE NOT DEFECT DATA TABLE

                                DataTable dtNotDefect = new DataTable();


                                dtNotDefect.Columns.Add("FirstName", typeof(string));
                                dtNotDefect.Columns.Add("LastName", typeof(string));
                                dtNotDefect.Columns.Add("Title", typeof(string));
                                


                                ActusLibrary.Company objCompany = ActusLibrary.Company.GetCompanyDetailsForAppraisalBasedOn(compId);

                                if (myExcelTable.Rows.Count > 0)
                                {
                                    int counter = 0;

                                    lstDepartments = new List<Departments>();
                                    lstDepartments = ActusLibrary.Departments.GetDepartmentList(compId);

                                    lstBusinessUnits = new List<BusinessUnits>();
                                    lstBusinessUnits = ActusLibrary.BusinessUnits.GetBusinessUnitsListByCompanyId(compId);

                                    lstLocations = new List<Locations>();
                                    lstLocations = ActusLibrary.Locations.GetLocationsListByCompanyId(compId);

                                    lstTeams = new List<Teams>();
                                    lstTeams = ActusLibrary.Teams.GetTeamsByCompanyId(compId);

                                    listEmployeeTypes = new Dictionary<string, string>();
                                    listEmployeeTypes = DataHelper.GetEmployeeTypes();

                                    listEmployeeGrades = new Dictionary<string, string>();
                                    listEmployeeGrades = DataHelper.GetEmployeeGrades();

                                    foreach (DataRow drExcelFileInfo in myExcelTable.Rows)
                                    {
                                        //ProcessEachExcelFileRecord(drExcelFileInfo, ref dtDefect, ref dtNotDefect, ref myDBTable, ref myManagersTable);                                    
                                        if (objCompany != null)
                                        {
                                            if (counter != 0)
                                            {
                                                ProcessEachExcelFileRecord(drExcelFileInfo, ref dtDefect, ref dtNotDefect, compId, ref myManagersTable, objCompany);
                                            }
                                            counter++;
                                        }
                                    }
                                }

                                if (dtDefect.Rows.Count > 0)
                                {
                                    divRecDefectedNotFound.Visible = false;
                                    panHeaderDefected.Visible = true;

                                    divExportToCSVDefetedRecords.Visible = true;
                                    grdDefectRecord.DataSource = dtDefect;
                                    grdDefectRecord.DataBind();
                                    ViewState["Defect"] = dtDefect;
                                }
                                else
                                {
                                    divRecDefectedNotFound.Visible = true;
                                    panHeaderDefected.Visible = false;
                                    panBodyDefected.Attributes.Add("style", "display:none");

                                    divExportToCSVDefetedRecords.Visible = false;
                                    grdDefectRecord.DataSource = null;
                                    grdDefectRecord.DataBind();
                                    ViewState["Defect"] = null;

                                }

                                if (dtNotDefect.Rows.Count > 0)
                                {
                                    divRecNotDefectedNotFound.Visible = false;
                                    panHeaderNotDefected.Visible = true;

                                    divExToCSVUploadedRecords.Visible = false;
                                    divButtonProceed.Visible = true;
                                    grdNotDefect.DataSource = dtNotDefect;
                                    grdNotDefect.DataBind();
                                    ViewState["NotDefect"] = dtNotDefect;
                                }
                                else
                                {
                                    divRecNotDefectedNotFound.Visible = true;
                                    panHeaderNotDefected.Visible = false;
                                    panBodyNotDefected.Attributes.Add("style", "display:none");

                                    divButtonProceed.Visible = false;
                                    divExToCSVUploadedRecords.Visible = false;
                                    grdNotDefect.DataSource = null;
                                    grdNotDefect.DataBind();
                                    ViewState["NotDefect"] = null;
                                }
                            }
                            else
                            {
                                lblMessage.Visible = true;
                                lblMessage.Text = "Columns are not in specified format.";
                                lblMessage.ForeColor = System.Drawing.Color.Red;
                            }
                        }
                        else
                        {
                            lblMessage.Visible = true;
                            lblMessage.Text = "Columns are not in specified format.";
                            lblMessage.ForeColor = System.Drawing.Color.Red;
                        }
                    }
                    else
                    {
                        lblMessage.Text = "File must be in .txt or .csv format delimited with ';'.";
                        lblMessage.ForeColor = System.Drawing.Color.Red;
                    }
                }
            }
            
        }



protected void ProcessEachExcelFileRecord(DataRow drExcelFileInfo, ref DataTable dtdefect, ref DataTable dtNotDefect,
        int companyId, ref DataTable myManagersTable, ActusLibrary.Company objCompanyAppraisal)
        {
            string FirstName = drExcelFileInfo["FirstName"].ToString();
            string LastName = drExcelFileInfo["LastName"].ToString();
            string Title = drExcelFileInfo["Title"].ToString();
            string strexists = string.Empty;
            string strissues = string.Empty;
            string strfname = string.Empty;
            string strlname = string.Empty;
            string stremail = string.Empty;
            string strjobtitle = string.Empty;
            string strusername = string.Empty;

            //Validation of employee fields

            //FirstName
            if (string.IsNullOrEmpty(FirstName.Trim()))
            {
                strissues = strissues + "FirstName missing" + ";";
            }

            //LastName
            if (string.IsNullOrEmpty(LastName.Trim()))
            {
                strissues = strissues + "LastName missing" + ";";
            }

            //Title
            if (!string.IsNullOrEmpty(Title.Trim()))
            {
                if (!Enum.IsDefined(typeof(DataHelper.EmployeeTitle), Title.Trim()))
                {
                    strissues = strissues + "Title is not valid" + ";";
                }
            }

            

                Employee objUploadedEmployee = ActusLibrary.Employee.GetEmployeeDetailsByUsername(Username,ApConfig.CurrentUser.CompId);

                if (!string.IsNullOrEmpty(Manager_Username))
                {
                    if (objUploadedEmployee != null) //Exisisting Employee
                    {
                        int? ManagerId = ActusLibrary.Employee.GetEmployeeDetailsByUsername(Manager_Username,ApConfig.CurrentUser.CompId).EmpId;

                        if (Rights.ToLower() == "manager")
                        {
                            //lstDirectReportee = objDirectReportee.GetDitectReporteeForTree(Convert.ToInt32(objUploadedEmployee.EmpId))
                            //                       .Select(m => Convert.ToInt32(m.EmpId)).ToList();

                            lstDirectReportee = objDirectReportee.GetCompleteEmployeesManagerDownlineTree(Convert.ToInt32(objUploadedEmployee.EmpId), Convert.ToInt32(ApConfig.CurrentUser.CompId), "manager")
                                                                                            .Where(e => e.RoleName.ToLower() == "manager")
                                                                                            .Select(x => Convert.ToInt32(x.EmpId)).ToList();

                            if ((lstDirectReportee != null) && (lstDirectReportee.Count > 0))
                            {
                                if (lstDirectReportee.Contains(Convert.ToInt32(ManagerId)))
                                {
                                    strissues = strissues + "you can not set your reportee as a manager." + ",";
                                }
                            }
                        }
                    }
                }

            }

            //Alternate Manager updattion
            if (isInvalidUsername == false && isInvalidAlternateManagername == false && isInvalidRights == false)
            {

                Employee objDirectReportee = new Employee();
                List<int> lstDirectReportee = new List<int>();

                if (!string.IsNullOrEmpty(Alternate_Manager_Username))
                {
                    if (Username == Alternate_Manager_Username)
                    {
                        strissues = strissues + "You can not set yourself as an alternative manager" + ",";
                    }
                }


            }

            if (strissues != "")
            {
                DataRow dr = dtdefect.NewRow();

                dr["FirstName"] = FirstName;
                dr["LastName"] = LastName;
                dr["Title"] = Title;
                
                dr["Issues"] = strissues;

                dtdefect.Rows.Add(dr);
            }
            else
            {
                DataRow drNot = dtNotDefect.NewRow();
                drNot["FirstName"] = FirstName;
                drNot["LastName"] = LastName;
                drNot["Title"] = Title;

                dtNotDefect.Rows.Add(drNot);
            }
        }



chnage the code as per your requirment , i have just extracted part of the code for your understanding purpose
 
Share this answer
 
v2

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)



CodeProject, 20 Bay Street, 11th Floor Toronto, Ontario, Canada M5J 2N8 +1 (416) 849-8900