C# SQLite file import prevent duplicates
- by jakesankey
Hi, I am attempting to get a directory (which is ever-growing) full of .txt comma delimited files to import into my SQLite db. I now have all of the files importing ok, however I need to have some way of excluding the files that have been previously added to db. I have a column in the db called FileName where the name and extension are stored next to each record from each file. Now I need to say 'If the code finds XXX.txt and XXX.txt is already in db, then skip this file'. Can I somehow add this logic to the getfiles command or is there another easy way?
using (SQLiteCommand insertCommand = con.CreateCommand())
{
SQLiteCommand cmdd = con.CreateCommand();
string[] files = Directory.GetFiles(@"C:\Documents and Settings\js91162\Desktop\", "R303717*.txt*", SearchOption.AllDirectories);
foreach (string file in files)
{
string FileNameExt1 = Path.GetFileName(file);
cmdd.CommandText =
@"
SELECT COUNT(*) FROM Import WHERE FileName = @FileExt;";
cmdd.Parameters.Add(new SQLiteParameter("@FileExt", FileNameExt1));
int count = Convert.ToInt32(cmdd.ExecuteScalar());
//int count = ((IConvertible)insertCommand.ExecuteScalar().ToInt32(null));
if (count == 0)
{
Console.WriteLine("Parsing CMM data for SQL database... Please wait.");
insertCommand.CommandText =
@"
INSERT INTO Import (FeatType, FeatName, Value, Actual, Nominal, Dev, TolMin, TolPlus, OutOfTol, PartNumber, CMMNumber, Date, FileName)
VALUES (@FeatType, @FeatName, @Value, @Actual, @Nominal, @Dev, @TolMin, @TolPlus, @OutOfTol, @PartNumber, @CMMNumber, @Date, @FileName);";
insertCommand.Parameters.Add(new SQLiteParameter("@FeatType", DbType.String));
insertCommand.Parameters.Add(new SQLiteParameter("@FeatName", DbType.String));
insertCommand.Parameters.Add(new SQLiteParameter("@Value", DbType.String));
insertCommand.Parameters.Add(new SQLiteParameter("@Actual", DbType.Decimal));
insertCommand.Parameters.Add(new SQLiteParameter("@Nominal", DbType.Decimal));
insertCommand.Parameters.Add(new SQLiteParameter("@Dev", DbType.Decimal));
insertCommand.Parameters.Add(new SQLiteParameter("@TolMin", DbType.Decimal));
insertCommand.Parameters.Add(new SQLiteParameter("@TolPlus", DbType.Decimal));
insertCommand.Parameters.Add(new SQLiteParameter("@OutOfTol", DbType.Decimal));
insertCommand.Parameters.Add(new SQLiteParameter("@Comment", DbType.String));
string FileNameExt = Path.GetFileName(file);
string RNumber = Path.GetFileNameWithoutExtension(file);
string RNumberE = RNumber.Split('_')[0];
string RNumberD = RNumber.Split('_')[1];
string RNumberDate = RNumber.Split('_')[2];
DateTime dateTime = DateTime.ParseExact(RNumberDate, "yyyyMMdd", Thread.CurrentThread.CurrentCulture);
string cmmDate = dateTime.ToString("dd-MMM-yyyy");
string[] lines = File.ReadAllLines(file);
bool parse = false;
foreach (string tmpLine in lines)
{
string line = tmpLine.Trim();
if (!parse && line.StartsWith("Feat. Type,"))
{
parse = true;
continue;
}
if (!parse || string.IsNullOrEmpty(line))
{
continue;
}
Console.WriteLine(tmpLine);
foreach (SQLiteParameter parameter in insertCommand.Parameters)
{
parameter.Value = null;
}
string[] values = line.Split(new[] { ',' });
for (int i = 0; i < values.Length - 1; i++)
{
SQLiteParameter param = insertCommand.Parameters[i];
if (param.DbType == DbType.Decimal)
{
decimal value;
param.Value = decimal.TryParse(values[i], out value) ? value : 0;
}
else
{
param.Value = values[i];
}
}
insertCommand.Parameters.Add(new SQLiteParameter("@PartNumber", RNumberE));
insertCommand.Parameters.Add(new SQLiteParameter("@CMMNumber", RNumberD));
insertCommand.Parameters.Add(new SQLiteParameter("@Date", cmmDate));
insertCommand.Parameters.Add(new SQLiteParameter("@FileName", FileNameExt));
//
insertCommand.ExecuteNonQuery();
}
}
}
Console.WriteLine("CMM data successfully imported to SQL database...");
}
con.Close();
}
