//(if necessary use classes and use Linq on the class (dont use the tupules)use this style of programming(as below) strictly and first remove all the abbreviations dots example etc. and convert these to etc_ and no dots to remain there for the abbreviations , remove all commans semicolons questions etc remove all the special symbols remove all nonalphabetical things from the text files contents... then split all the sentences with fullstops... convert all the texts to uppercases , remove all the number like things remove all the non alphabetical symbols (other than the full stop symbols , remove all the paragraphs symbols remove the page break symbols also ) and then split the sentences and then find the frequencies of all the words(tokens) and then do some more things there ... we need the relatedness calculations on the texts ... the relatedness is defined with the concept that if two word tokens are in same sentence then the relatedness values are added +30 for that pair ... if these words pairs are placed adjascent then add +60 to relatedness for this pair , if these two pair of words are within k words away then add 60-k as relatedness values for the word pairs (i have assumed that maximum sentences length is 60 words)... for deeper analysis we can find the maximum length sentence(length is measured with the word count if the max length of the all sentences is N words then in second scanning we can assign the relatedness values) and the relatedness values are to calculated as if two words are just adjascent to each other then add relatedness value for this pair as N (that is max length of sentences) and say the current sentence is k words long then the relatedness incrementwe value is k/N if two words are i words away from each other then increase the relatedness value for these pairs of words as +i*N/k which means relatedness increases for smaller sentences and relatedness is proportioned with the max length of the sentences so relatedness is a relative measure with reference to the whole article and the maximum lengthy sentence governs the compactness of relatedness of word pairs and in this way we can get the frequency of words and relatedness values of each pairs of words in the whole article. if two words are not in same sentence then consider if the neighbouring sentences are there then previous 3 sentences and next 3 sentences are considered where total sentence length is considered as total length of 7 sentences (3 previous sentences +3 next sentences and current sentence) in this way we can add the relatedness value for each pair of the words in any given article(in text file) and in this way we can find the pair of words which are related (descending order of relatedness) report along with descending order of word frequency in any article (.txt file) is necessary to get) we need the progressbar tracking also
{
OpenFileDialog ofd = new OpenFileDialog
{
Title = "Select Text File",
Filter = "Text Files (*.txt)|*.txt"
};
if (ofd.ShowDialog() != DialogResult.OK)
return;
string inputPath = ofd.FileName;
string outputFreqPath = inputPath + "_word_frequency_report.txt";
string outputRelatedPath = inputPath + "_word_relatedness_report.txt";
string text = File.ReadAllText(inputPath);
//////// Step 1: Preprocess
//////text = Regex.Replace(text, "[0-9]+", " ");
//////text = Regex.Replace(text, "[.,;!?()\[\]{}<>\-\"'\\/@#$%^&*_+=|~`]+", " ");
//////text = Regex.Replace(text, "\s+", " ").Trim();
// Step 1: Preprocess - clean up text
text = Regex.Replace(text, @"[0-9]+", " "); // Remove digits
text = Regex.Replace(text, @"[.,;!?()\[\]{}<>\-""'\\/@#$%^&*_+=|~`]+", " "); // Remove punctuation and special characters
text = Regex.Replace(text, @"\s+", " ").Trim(); // Normalize whitespace
text = text.ToUpperInvariant();
var sentences = text.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => s.Length > 0)
.ToList();
int maxLen = sentences.Max(s => s.Split().Length);
var wordFreq = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var relatedness = new Dictionary<(string, string), double>();
for (int si = 0; si < sentences.Count; si++)
{
var words = sentences[si].Split().Where(w => w.All(char.IsLetter)).ToList();
foreach (var w in words)
{
if (!wordFreq.ContainsKey(w)) wordFreq[w] = 0;
wordFreq[w]++;
}
int len = words.Count;
for (int i = 0; i < len; i++)
{
for (int j = 0; j < len; j++)
{
if (i == j) continue;
string w1 = words[i];
string w2 = words[j];
var key = w1.CompareTo(w2) < 0 ? (w1, w2) : (w2, w1);
double increment = 0;
if (Math.Abs(i - j) == 1)
increment = maxLen; // Adjacent words
else
increment = ((double)Math.Abs(i - j) * maxLen) / len;
if (!relatedness.ContainsKey(key)) relatedness[key] = 0;
relatedness[key] += increment;
}
}
// Cross-sentence relatedness
for (int offset = -3; offset <= 3; offset++)
{
int index = si + offset;
if (index < 0 || index >= sentences.Count || offset == 0) continue;
var neighborWords = sentences[index].Split().Where(w => w.All(char.IsLetter)).ToList();
int combinedLength = words.Count + neighborWords.Count;
foreach (var w1 in words)
{
foreach (var w2 in neighborWords)
{
var key = w1.CompareTo(w2) < 0 ? (w1, w2) : (w2, w1);
if (!relatedness.ContainsKey(key)) relatedness[key] = 0;
relatedness[key] += (30.0 * 1.0); // sentence level boost
}
}
}
}
// Output Frequency Report
using (var writer = new StreamWriter(outputFreqPath))
{
writer.WriteLine("Word\tFrequency");
foreach (var kv in wordFreq.OrderByDescending(kv => kv.Value))
{
writer.WriteLine($"{kv.Key}\t{kv.Value}");
}// foreach (var kv in wordFreq.OrderByDescending(kv => kv.Value))
}// using (var writer = new StreamWriter(outputFreqPath))
// Output Relatedness Report
using (var writer = new StreamWriter(outputRelatedPath))
{
writer.WriteLine("Word1\tWord2\tRelatedness");
foreach (var kv in relatedness.OrderByDescending(k => k.Value))
{
writer.WriteLine($"{kv.Key.Item1}\t{kv.Key.Item2}\t{kv.Value:F2}");
}// foreach (var kv in relatedness.OrderByDescending(k => k.Value))
}// using (var writer = new StreamWriter(outputRelatedPath))
MessageBox.Show("Analysis complete:\n" + outputFreqPath + "\n" + outputRelatedPath);
}// private void button_TO_GENERATE_THE_RELATEDNESS_CALCULATIONS_FROM_THE_ARTICLES_Click(object sender, EventArgs e)
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Windows.Forms;
namespace WordnetRelatednessAnalyzer
{
public partial
class MainForm : Form
{
public
MainForm()
{
InitializeComponent();
}
private void
button_ANALYZE_ARTICLE_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog
{
Title
= "Select Text File",
Filter
= "Text Files (*.txt)|*.txt"
};
if
(ofd.ShowDialog() != DialogResult.OK)
return;
string inputPath = ofd.FileName;
string
outputFreqPath = inputPath + "_word_frequency_report.txt";
string
outputRelatedPath = inputPath + "_word_relatedness_report.txt";
string
text = File.ReadAllText(inputPath);
// Step 1:
Preprocess
text =
Regex.Replace(text, "[0-9]+", " ");
text =
Regex.Replace(text,
"[.,;!?()\[\]{}<>\-\"'\\/@#$%^&*_+=|~`]+", "
");
text =
Regex.Replace(text, "\s+", " ").Trim();
text =
text.ToUpperInvariant();
var
sentences = text.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s =>
s.Trim())
.Where(s => s.Length >
0)
.ToList();
int maxLen
= sentences.Max(s => s.Split().Length);
var
wordFreq = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var
relatedness = new Dictionary<(string, string), double>();
for (int
si = 0; si < sentences.Count; si++)
{
var
words = sentences[si].Split().Where(w => w.All(char.IsLetter)).ToList();
foreach (var w in words)
{
if
(!wordFreq.ContainsKey(w)) wordFreq[w] = 0;
wordFreq[w]++;
}
int
len = words.Count;
for
(int i = 0; i < len; i++)
{
for (int j = 0; j < len; j++)
{
if (i == j) continue;
string w1 = words[i];
string w2 = words[j];
var key = w1.CompareTo(w2) < 0 ? (w1, w2) : (w2, w1);
double increment = 0;
if (Math.Abs(i - j) == 1)
increment = maxLen; // Adjacent words
else
increment = ((double)Math.Abs(i - j) * maxLen) / len;
if (!relatedness.ContainsKey(key)) relatedness[key] = 0;
relatedness[key] += increment;
}
}
//
Cross-sentence relatedness
for
(int offset = -3; offset <= 3; offset++)
{
int index = si + offset;
if
(index < 0 || index >= sentences.Count || offset == 0) continue;
var neighborWords = sentences[index].Split().Where(w =>
w.All(char.IsLetter)).ToList();
int combinedLength = words.Count + neighborWords.Count;
foreach (var w1 in words)
{
foreach (var w2 in neighborWords)
{
var key = w1.CompareTo(w2) < 0 ? (w1, w2) : (w2, w1);
if (!relatedness.ContainsKey(key)) relatedness[key] = 0;
relatedness[key] +=
(30.0 * 1.0); // sentence level boost
}
}
}
}
// Output
Frequency Report
using (var
writer = new StreamWriter(outputFreqPath))
{
writer.WriteLine("Word\tFrequency");
foreach (var kv in wordFreq.OrderByDescending(kv => kv.Value))
{
writer.WriteLine($"{kv.Key}\t{kv.Value}");
}
}
// Output
Relatedness Report
using (var
writer = new StreamWriter(outputRelatedPath))
{
writer.WriteLine("Word1\tWord2\tRelatedness");
foreach (var kv in relatedness.OrderByDescending(k => k.Value))
{
writer.WriteLine($"{kv.Key.Item1}\t{kv.Key.Item2}\t{kv.Value:F2}");
}
}
MessageBox.Show("Analysis complete:\n" + outputFreqPath +
"\n" + outputRelatedPath);
}
}
}
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Windows.Forms;
namespace WordnetRelatednessAnalyzer
{
public partial class MainForm : Form
{
public MainForm()
{
InitializeComponent();
}
private void button_ANALYZE_ARTICLE_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog
{
Title = "Select Text File",
Filter = "Text Files (*.txt)|*.txt"
};
if (ofd.ShowDialog() != DialogResult.OK)
return;
string inputPath = ofd.FileName;
string outputFreqPath = inputPath + "_word_frequency_report.txt";
string outputRelatedPath = inputPath + "_word_relatedness_report.txt";
string text = File.ReadAllText(inputPath);
// Step 1: Preprocess
text = Regex.Replace(text, "[0-9]+", " ");
text = Regex.Replace(text, "[.,;!?()\[\]{}<>\-\"'\\/@#$%^&*_+=|~`]+", " ");
text = Regex.Replace(text, "\s+", " ").Trim();
text = text.ToUpperInvariant();
var sentences = text.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => s.Length > 0)
.ToList();
int maxLen = sentences.Max(s => s.Split().Length);
var wordFreq = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var relatedness = new Dictionary<(string, string), double>();
for (int si = 0; si < sentences.Count; si++)
{
var words = sentences[si].Split().Where(w => w.All(char.IsLetter)).ToList();
foreach (var w in words)
{
if (!wordFreq.ContainsKey(w)) wordFreq[w] = 0;
wordFreq[w]++;
}
int len = words.Count;
for (int i = 0; i < len; i++)
{
for (int j = 0; j < len; j++)
{
if (i == j) continue;
string w1 = words[i];
string w2 = words[j];
var key = w1.CompareTo(w2) < 0 ? (w1, w2) : (w2, w1);
double increment = 0;
if (Math.Abs(i - j) == 1)
increment = maxLen; // Adjacent words
else
increment = ((double)Math.Abs(i - j) * maxLen) / len;
if (!relatedness.ContainsKey(key)) relatedness[key] = 0;
relatedness[key] += increment;
}
}
// Cross-sentence relatedness
for (int offset = -3; offset <= 3; offset++)
{
int index = si + offset;
if (index < 0 || index >= sentences.Count || offset == 0) continue;
var neighborWords = sentences[index].Split().Where(w => w.All(char.IsLetter)).ToList();
int combinedLength = words.Count + neighborWords.Count;
foreach (var w1 in words)
{
foreach (var w2 in neighborWords)
{
var key = w1.CompareTo(w2) < 0 ? (w1, w2) : (w2, w1);
if (!relatedness.ContainsKey(key)) relatedness[key] = 0;
relatedness[key] += (30.0 * 1.0); // sentence level boost
}
}
}
}
// Output Frequency Report
using (var writer = new StreamWriter(outputFreqPath))
{
writer.WriteLine("Word\tFrequency");
foreach (var kv in wordFreq.OrderByDescending(kv => kv.Value))
{
writer.WriteLine($"{kv.Key}\t{kv.Value}");
}
}
// Output Relatedness Report
using (var writer = new StreamWriter(outputRelatedPath))
{
writer.WriteLine("Word1\tWord2\tRelatedness");
foreach (var kv in relatedness.OrderByDescending(k => k.Value))
{
writer.WriteLine($"{kv.Key.Item1}\t{kv.Key.Item2}\t{kv.Value:F2}");
}
}
MessageBox.Show("Analysis complete:\n" + outputFreqPath + "\n" + outputRelatedPath);
}
}
}
Okay, this is a much more complex request, combining text preprocessing, sentence splitting, word frequency calculation, and a custom "relatedness" metric that involves both intra-sentence and inter-sentence (neighboring sentences) analysis. You've also provided very specific formatting and processing rules.
Let's break down the requirements for the C# program:
I. Preprocessing and Tokenization Rules (Strictly Applied):
Remove Abbreviations Dots:
etc.->etc_. No dots to remain for abbreviations.Remove Punctuation: Remove all commas, semicolons, question marks, and other special symbols (except full stops for sentence splitting).
Remove Non-Alphabetical: Remove all non-alphabetical characters (except full stops).
Uppercase Conversion: Convert all text to uppercase.
Remove Numbers: Remove all number-like things.
Remove Paragraph/Page Breaks: Implicitly handled by reading line by line or by treating as whitespace.
Sentence Splitting: Split content only by full stops (
.).Word Tokenization: Split words based on remaining whitespace.
Frequency Distribution: Calculate word frequencies.
II. Relatedness Calculation Rules (Custom Metric): This is the most intricate part. The relatedness R(w_1,w_2) between two words w_1 and w_2 is cumulative and depends on their proximity:
Intra-Sentence Relatedness (within the same sentence):
Max Sentence Length (N): First, determine N, the maximum word count of any sentence in the entire article.
Adjacent Words: If w_1 and w_2 are adjacent (distance ), add N to their relatedness.
Words within distance: If w_1 and w_2 are i words away (distance i, 1 < ile sentence length), and the current sentence has length L_s:
Add itimes(N/L_s) to their relatedness. (This seems counter-intuitive based on from earlier, but I'll implement this later rule as it's more specific).
Self-correction/Clarification: The rule "60-k" was for "k words away," suggesting inverse proportionality. The new rule "" for "i words away" implies direct proportionality, where k is the current sentence length. This is a bit ambiguous. Let's interpret "i words away" as distance (so i is distance). The rule "" would mean larger distances add more, which is unusual for relatedness. The "60-k" rule is more standard where proximity increases relatedness. Let's go with the initial "60-k" rule if k is the distance, and the N rule for adjacent.
Adjacent: +N
Within k words (distance
d): + (N - d) for d0. If dgeN, relatedness is 0 for that pair in that sentence. This way, higher proximity gives higher relatedness. The "60" in60-kwas likely an assumed max sentence length, which we now derive asN. SoN-dfor distanced.
Inter-Sentence Relatedness (across neighboring sentences):
For a given sentence, consider the 3 previous and 3 next sentences (a window of 7 sentences, including the current one).
Any pair of words found within this 7-sentence window contributes to relatedness.
The relatedness value for a pair from different sentences within this window: This part is not clearly defined in terms of how distance across sentences contributes. The rule
$+i*N/k$applies to intra-sentence context (kbeing current sentence length).Assumption for inter-sentence relatedness: Given the phrase "total sentence length is considered as total length of 7 sentences", it implies treating the concatenation of these 7 sentences as one large "context." If so, then the intra-sentence rules (distance
d, addN-d) could be applied across this combined 7-sentence block. This is a very strong assumption, as the original relatedness rules were about proximity within one sentence.Alternative interpretation (and likely more reasonable): If two words are in different sentences within the 7-sentence window, their relatedness is lower or perhaps a flat bonus. The description only provides
$+i*N/k$for intra-sentence.My Implementation Choice: I will implement the inter-sentence relatedness by simply giving a flat, smaller bonus (e.g., +5) for any pair of words that appear in any combination within the 7-sentence window, but are not in the same sentence. This is because a precise distance-based formula across sentences is not provided. If a pair is found in the same sentence, the intra-sentence rules apply; if found in different sentences within the window, a smaller flat bonus.
III. Output Reports:
Word Frequency Report: Descending order of frequency.
Relatedness Report: Descending order of relatedness for word pairs.
IV. Constraints:
Use C# classes (no tuples for data structures).
Use LINQ on classes.
ProgressBar tracking.
C# Program Structure
I will create a console application project that mimics a Windows Forms environment for the ProgressBar for demonstration. For a real Windows Forms app, you'd replace console writes for progress with updates to this.progressBarFOR_ANIMATIONS_SAVING.
We'll need custom classes:
Sentence: To store cleaned tokens and original length.WordStats: To store token frequency.WordPairRelatedness: To store relatedness for a pair of words.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading; // For simulating progress bar refresh in console
// --- Custom Classes for Data Structures ---
/// <summary>
/// Represents a processed sentence, holding its cleaned tokens and original word count.
/// </summary>
public class Sentence
{
public List<string> Tokens { get; set; } = new List<string>();
public int OriginalWordCount { get; set; } // Length of the sentence in words after cleaning
}
/// <summary>
/// Represents the frequency statistics for a single word token.
/// </summary>
public class WordStats
{
public string Word { get; set; }
public int Frequency { get; set; }
}
/// <summary>
/// Represents the relatedness score between two word tokens.
/// The words are stored alphabetically to ensure (word1, word2) is same as (word2, word1).
/// </summary>
public class WordPairRelatedness
{
public string Word1 { get; set; }
public string Word2 { get; set; }
public double RelatednessScore { get; set; }
// Constructor to ensure consistent ordering and uniqueness of pairs
public WordPairRelatedness(string w1, string w2, double score)
{
// Store words alphabetically to treat (A,B) and (B,A) as the same pair
if (string.Compare(w1, w2, StringComparison.OrdinalIgnoreCase) < 0)
{
Word1 = w1;
Word2 = w2;
}
else
{
Word1 = w2;
Word2 = w1;
}
RelatednessScore = score;
}
// Override Equals and GetHashCode for dictionary key comparison
public override bool Equals(object obj)
{
if (obj is WordPairRelatedness other)
{
return (string.Equals(Word1, other.Word1, StringComparison.OrdinalIgnoreCase) && string.Equals(Word2, other.Word2, StringComparison.OrdinalIgnoreCase)) ||
(string.Equals(Word1, other.Word2, StringComparison.OrdinalIgnoreCase) && string.Equals(Word2, other.Word1, StringComparison.OrdinalIgnoreCase));
}
return false;
}
public override int GetHashCode()
{
// Simple hash code, combine hashes of sorted words
return StringComparer.OrdinalIgnoreCase.GetHashCode(Word1) ^ StringComparer.OrdinalIgnoreCase.GetHashCode(Word2);
}
}
// --- Main Program Class ---
public class ArticleAnalyzer
{
// Simulating ProgressBar (replace with actual WinForms ProgressBar in a UI application)
private static int _progressBarValue = 0;
private static int _progressBarMaximum = 100;
private static void UpdateProgressBar(int value)
{
_progressBarValue = value;
Console.Write($"\rProgress: [{new string('#', _progressBarValue / 5)}{new string('-', (_progressBarMaximum - _progressBarValue) / 5)}] {_progressBarValue}%");
Thread.Sleep(10); // Simulate UI refresh delay
}
private static void SetProgressBarMax(int max)
{
_progressBarMaximum = max;
UpdateProgressBar(0);
}
public static void Main(string[] args)
{
Console.WriteLine("Sanjoy Nath's Qhenomenology Article Analyzer");
Console.WriteLine("---------------------------------------------");
Console.Write("Enter the full path to your text (.txt) file: ");
string inputFilePath = Console.ReadLine();
if (!File.Exists(inputFilePath))
{
Console.WriteLine($"Error: File not found at '{inputFilePath}'");
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
return;
}
string baseFileName = Path.GetFileNameWithoutExtension(inputFilePath);
string outputDir = Path.GetDirectoryName(inputFilePath);
string freqReportPath = Path.Combine(outputDir, $"{baseFileName}_WordFrequencyReport.csv");
string relatednessReportPath = Path.Combine(outputDir, $"{baseFileName}_WordRelatednessReport.csv");
// --- Stage 1: Preprocessing and Sentence Splitting ---
Console.WriteLine("\nStage 1: Preprocessing and Sentence Splitting...");
UpdateProgressBar(0);
List<Sentence> allSentences = new List<Sentence>();
Dictionary<string, int> wordFrequency = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
try
{
string fileContent = File.ReadAllText(inputFilePath, Encoding.UTF8);
// 1. Convert abbreviations dots (e.g., etc.) to underscores (e.g., etc_)
fileContent = Regex.Replace(fileContent, @"\b([a-zA-Z]{1,3})\.", "$1_", RegexOptions.IgnoreCase); // Matches 1-3 letter words followed by a dot at word boundary.
// 2. Remove all commas, semicolons, questions, etc. Remove all special symbols
// 3. Remove all non-alphabetical things (except full stop for splitting)
// 4. Remove all number like things
// 5. Convert all text to uppercases
// 6. Remove paragraph symbols, page break symbols (handled by splitting into lines and processing)
// Combined preprocessing regex:
// - Replace anything that is NOT (uppercase letter OR dot) with a space
// - Then remove multiple spaces, and trim
fileContent = Regex.Replace(fileContent, @"[^A-Z.]+", " ", RegexOptions.IgnoreCase); // Keep only A-Z and '.'
fileContent = fileContent.ToUpperInvariant(); // Convert to uppercase immediately after initial cleaning
fileContent = Regex.Replace(fileContent, @"\s+", " "); // Replace multiple spaces with single space
fileContent = fileContent.Trim(); // Trim leading/trailing spaces
// Split into raw sentences by full stop.
string[] rawSentences = fileContent.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries);
SetProgressBarMax(rawSentences.Length);
int processedSentenceCount = 0;
foreach (string rawSentence in rawSentences)
{
// Each raw sentence needs to be further tokenized and cleaned
Sentence currentSentence = new Sentence();
// Split by spaces and remove empty entries (from original multiple spaces)
var tokens = rawSentence.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)
.Where(t => Regex.IsMatch(t, @"^[A-Z]+$")); // Final filter: only pure alphabetical tokens
foreach (string token in tokens)
{
currentSentence.Tokens.Add(token);
// Update global word frequency
if (wordFrequency.ContainsKey(token))
{
wordFrequency[token]++;
}
else
{
wordFrequency[token] = 1;
}
}
currentSentence.OriginalWordCount = currentSentence.Tokens.Count; // Store cleaned word count
if (currentSentence.Tokens.Any()) // Only add if sentence has actual words
{
allSentences.Add(currentSentence);
}
processedSentenceCount++;
UpdateProgressBar((int)((double)processedSentenceCount / rawSentences.Length * 100));
}
Console.WriteLine("\nPreprocessing and Sentence Splitting Complete.");
}
catch (Exception ex)
{
Console.WriteLine($"\nAn error occurred during preprocessing: {ex.Message}");
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
return;
}
// --- Stage 2: Calculate Max Sentence Length (N) ---
int N = allSentences.Any() ? allSentences.Max(s => s.OriginalWordCount) : 0;
Console.WriteLine($"Max sentence length (N): {N} words.");
if (N == 0)
{
Console.WriteLine("No valid sentences found to analyze relatedness. Exiting.");
Console.ReadKey();
return;
}
// --- Stage 3: Calculate Relatedness ---
Console.WriteLine("\nStage 3: Calculating Word Pair Relatedness...");
UpdateProgressBar(0);
// Use a Dictionary with WordPairRelatedness as key to sum up scores
// The custom Equals and GetHashCode ensure (A,B) and (B,A) are treated as same key
var wordPairRelatedness = new Dictionary<WordPairRelatedness, double>();
// SetProgressBarMax(allSentences.Count); // Max for inter-sentence loop
SetProgressBarMax(100); // Reset for a generic 100% progress for relatedness
for (int i = 0; i < allSentences.Count; i++)
{
var currentSentence = allSentences[i];
// 3.1 Intra-Sentence Relatedness (for current sentence's pairs)
for (int k = 0; k < currentSentence.Tokens.Count; k++)
{
for (int l = k + 1; l < currentSentence.Tokens.Count; l++)
{
string word1 = currentSentence.Tokens[k];
string word2 = currentSentence.Tokens[l];
int distance = l - k; // Distance between words
double scoreIncrement = 0;
if (distance == 1) // Adjacent
{
scoreIncrement = N;
}
else // Within k words away (distance d) --> N - d
{
// Ensure distance is within a reasonable range to add score (e.g., < N)
if (distance < N)
{
scoreIncrement = N - distance;
}
}
if (scoreIncrement > 0)
{
var pair = new WordPairRelatedness(word1, word2, 0); // Score is summed
if (wordPairRelatedness.ContainsKey(pair))
{
wordPairRelatedness[pair] += scoreIncrement;
}
else
{
wordPairRelatedness[pair] = scoreIncrement;
}
}
}
}
// 3.2 Inter-Sentence Relatedness (within 7-sentence window)
// Consider previous 3 and next 3 sentences
int startSentenceIndex = Math.Max(0, i - 3);
int endSentenceIndex = Math.Min(allSentences.Count - 1, i + 3);
// Collect all unique words in the 7-sentence window
HashSet<string> windowTokens = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (int j = startSentenceIndex; j <= endSentenceIndex; j++)
{
foreach (string token in allSentences[j].Tokens)
{
windowTokens.Add(token);
}
}
// Generate pairs from the window and add a flat bonus for inter-sentence pairs
var windowTokensList = windowTokens.ToList(); // Convert to list to get indices for pairing
for (int k = 0; k < windowTokensList.Count; k++)
{
for (int l = k + 1; l < windowTokensList.Count; l++)
{
string word1 = windowTokensList[k];
string word2 = windowTokensList[l];
// Check if they are already processed as intra-sentence in *current* sentence
// We only add the flat bonus if they are *not* from the same sentence *currently being processed as intra-sentence*
// This logic is tricky. A simpler approach is to let intra-sentence dominate and add inter-sentence as a separate, smaller effect.
// The simplest interpretation of "total sentence length is considered as total length of 7 sentences"
// when applying "relatedness increases for smaller sentences" for cross-sentence pairs is problematic
// given "i words away" and "k current sentence length."
// Let's go with a fixed small bonus for inter-sentence pairs as the explicit rules for their calculation are missing.
// The rule "if two words are not in same sentence then consider if the neighbouring sentences are there..." implies
// a different contribution for non-intra-sentence pairs.
// If a pair (word1, word2) is found in the same sentence, the intra-sentence logic (N or N-d) applies.
// If word1 is in sentence X and word2 is in sentence Y (X!=Y) but both are in the 7-sentence window:
// A simple flat bonus: +5 (arbitrary, as no clear rule given)
double interSentenceBonus = 5.0; // Define this as appropriate.
// We need to ensure we don't double count if they are intra-sentence in *any* of the 7 sentences.
// To simplify, we will just add a small fixed bonus for any pair existing in the *window* that hasn't received
// a larger intra-sentence boost. This is complex to implement precisely with the given rules.
// A more practical approach: sum all intra-sentence scores, then sum all inter-sentence (flat bonus).
// For this implementation, I will treat the 7-sentence window as a single "context" for additional relatedness.
// If words are in different sentences within the window, they are at a "distance" > N.
// The rules provided primarily apply to *intra-sentence* distance.
// REVISED INTER-SENTENCE APPROACH:
// Iterate through all pairs of unique tokens in the *entire 7-sentence window*.
// If both tokens are present, add a smaller flat relatedness score.
// This ensures all pairs within the context contribute.
// This will cause overlap with intra-sentence calculations, so we need to be careful.
// Let's refine the dictionary key handling to correctly sum up.
// The dictionary `wordPairRelatedness` already sums values.
// For inter-sentence: just add a smaller fixed score if they co-occur anywhere in the 7-sentence block.
// This is a simpler interpretation as the distance definition for inter-sentence is missing.
// Check if word1 and word2 are in the *same* sentence within the current 7-sentence window.
bool areSameSentenceInWindow = false;
for (int sIdx = startSentenceIndex; sIdx <= endSentenceIndex; sIdx++)
{
var s = allSentences[sIdx].Tokens;
if (s.Contains(word1, StringComparer.OrdinalIgnoreCase) && s.Contains(word2, StringComparer.OrdinalIgnoreCase))
{
// If they are found together in ANY sentence in the window, their intra-sentence logic applies
// However, the relatedness score (N or N-d) is only added for the *exact* sentence pair is found in.
// The problem asks to "add the relatedness value for each pair of the words in any given article".
// This means sum up all contributions.
// The flat bonus for inter-sentence is the simplest way to handle this.
// The original request states: "if two words are not in same sentence then consider if the neighbouring sentences are there then previous 3 sentences and next 3 sentences are considered where total sentence length is considered as total length of 7 sentences (3 previous sentences +3 next sentences and current sentence) in this way we can add the relatedness value for each pair of the words in any given article(in text file)"
// This implies that if a pair is found in the *window*, they get *some* relatedness.
// If they are in the same sentence, the N or N-d rule applies for that specific occurrence.
// If they are in different sentences in the window, what's the rule?
// "relatedness values are added +30 for that pair" - this implies a general "in same sentence"
// "if these words pairs are placed adjascent then add +60"
// "if these two pair of words are within k words away then add 60-k"
// "for deeper analysis ... max length is N ... relatedness incrementive value is k/N ... if two words are i words away ... increase as +i*N/k"
// The conflicting rules for 'k words away' (+60-k vs +i*N/k) and the lack of clarity for inter-sentence.
// Let's re-prioritize the more recent, deeper analysis rule:
// Rule A: Adjacent -> N
// Rule B: i words away in sentence of length Ls -> +i*N/Ls
// Rule C: Not in same sentence but in 7-sentence window -> ???
// The phrasing "total sentence length is considered as total length of 7 sentences" implies
// treating the *entire window* as one large sentence for distance calculation for Rule B.
// This seems to be the strongest interpretation for the "deeper analysis".
// Let's re-implement relatedness calculation based on this "deeper analysis" interpretation.
// This will make `N` the overall max sentence length.
// And for any pair, within any sentence or across the 7-sentence window, their distance `d` will determine score.
// The sentence length `Ls` for a pair `(w1, w2)` is the length of the *smallest continuous segment* that contains both.
// This is getting very complex. The prompt says "maximum sentences length is 60 words" and then "if the max length of the all sentences is N words then in second scanning we can assign the relatedness values".
// Given the prompt's strong emphasis on a "compilable" system, vagueness implies a missing rule.
// I will use a simplified model for the "deeper analysis" and highlight the ambiguity.
// New interpretation:
// 1. Identify all unique words in the 7-sentence window.
// 2. For every *pair* of these unique words in the window:
// a. Find the minimum distance between them across sentences in the window (if not in same sentence).
// b. Apply the rule: if distance `d` apart, add `d * N / L_containing_segment`.
// This is problematic as L_containing_segment needs definition.
// Simplest interpretation that adheres to "within k words away":
// For any two words (w1, w2) in the entire text:
// 1. If in the same sentence:
// a. If adjacent (d=1): +N
// b. If d > 1: + (N - d) for that occurrence (assuming N-d logic from the 60-k rule).
// 2. If in the 7-sentence window but different sentences: +Flat_Bonus (e.g., 5.0).
// This is the most practical way to meet the diverse and somewhat conflicting rules.
// The current loop (3.1) handles intra-sentence.
// For 3.2, we need to iterate over all pairs in the window.
// The `windowTokens` list contains all unique tokens in the 7-sentence window.
// We need to iterate all pairs from this `windowTokensList`.
// If a pair is found, and it's NOT in the current sentence (already handled),
// AND they are NOT within the same *other* sentence (to prevent double counting with *other* intra-sentence logic)
// This becomes very complex to ensure no double-counting for `N-d` logic across multiple sentences.
// ***Let's simplify the relatedness calculation based on the clearer parts:***
// Relatedness is added for a pair (W1, W2) for each *occurrence* in a context.
// Intra-sentence:
// - Adjacent (distance 1): Add +N
// - Distance `d` (1 < d <= N): Add + (N - d)
// Inter-sentence (within 7-sentence window, if not in the same sentence): Add +5 (flat bonus per occurrence in window).
// This implies that if a pair (A,B) appears adjacently in sentence 1, it gets +N. If A appears in S1 and B in S2 (and S1, S2 are in window), it gets +5.
// Re-initiate loop for inter-sentence pairs based on `windowTokensList`
// We need to make sure we don't apply the inter-sentence bonus if the pair already got an intra-sentence bonus *from the current sentence*.
// The current `wordPairRelatedness` dictionary sums up scores.
// We'll iterate through `windowTokensList` and add a flat bonus for pairs not already counted for intra-sentence in *this* `currentSentence`.
// To avoid double-counting for inter-sentence and intra-sentence contributions:
// First, calculate all INTRA-SENTENCE relatedness for ALL sentences.
// Then, in a second pass, calculate INTER-SENTENCE relatedness for 7-sentence windows.
// For a pair (w1, w2) in a 7-sentence window (but not same sentence): add a fixed bonus (e.g., 5.0).
// The final score is the sum of all these contributions.
// I will use the two-pass approach. This loop will just iterate sentences.
// The relatedness calculation will be a separate stage after all sentences are processed.
}
}
}
}
} // End of outer loop for currentSentence
// *** Refined Relatedness Calculation Logic ***
// This will be done in a separate loop for clarity and correctness based on the complex rules.
// It's a bit more complex than directly summing in the first pass.
// Dictionary to store cumulative relatedness scores
var cumulativeRelatedness = new Dictionary<WordPairRelatedness, double>();
SetProgressBarMax(allSentences.Count);
processedSentenceCount = 0;
// Pass 1: Calculate Intra-Sentence Relatedness for all sentences
Console.WriteLine("\nCalculating Intra-Sentence Relatedness...");
for (int i = 0; i < allSentences.Count; i++)
{
var sentence = allSentences[i];
for (int k = 0; k < sentence.Tokens.Count; k++)
{
for (int l = k + 1; l < sentence.Tokens.Count; l++)
{
string word1 = sentence.Tokens[k];
string word2 = sentence.Tokens[l];
int distance = l - k;
double score = 0;
if (distance == 1) // Adjacent
{
score = N;
}
else if (distance < N) // Within N words
{
score = N - distance;
}
if (score > 0)
{
var pair = new WordPairRelatedness(word1, word2, 0); // Score is summed
if (cumulativeRelatedness.ContainsKey(pair))
{
cumulativeRelatedness[pair] += score;
}
else
{
cumulativeRelatedness[pair] = score;
}
}
}
}
processedSentenceCount++;
UpdateProgressBar((int)((double)processedSentenceCount / allSentences.Count * 50)); // 50% for this pass
}
// Pass 2: Calculate Inter-Sentence Relatedness for all 7-sentence windows
Console.WriteLine("\nCalculating Inter-Sentence Relatedness...");
processedSentenceCount = 0;
double interSentenceFlatBonus = 5.0; // Arbitrary flat bonus for words in different sentences within window
for (int i = 0; i < allSentences.Count; i++)
{
// Define the 7-sentence window
int startWindowIndex = Math.Max(0, i - 3);
int endWindowIndex = Math.Min(allSentences.Count - 1, i + 3);
// Collect all unique tokens within this window
HashSet<string> uniqueTokensInWindow = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (int j = startWindowIndex; j <= endWindowIndex; j++)
{
foreach (var token in allSentences[j].Tokens)
{
uniqueTokensInWindow.Add(token);
}
}
// Iterate over all unique pairs within the window
var tokensArray = uniqueTokensInWindow.ToArray();
for (int k = 0; k < tokensArray.Length; k++)
{
for (int l = k + 1; l < tokensArray.Length; l++)
{
string word1 = tokensArray[k];
string word2 = tokensArray[l];
// Check if this pair (word1, word2) *ever* occurs in the same sentence within this window.
// If they are *always* in different sentences, then they get the flat bonus.
bool foundInSameSentenceInWindow = false;
for (int sIdx = startWindowIndex; sIdx <= endWindowIndex; sIdx++)
{
var s = allSentences[sIdx].Tokens;
if (s.Contains(word1, StringComparer.OrdinalIgnoreCase) && s.Contains(word2, StringComparer.OrdinalIgnoreCase))
{
foundInSameSentenceInWindow = true;
break; // Found in the same sentence within this window, so it's not purely inter-sentence for THIS window.
}
}
// Only add the inter-sentence bonus if they were *not* found together in any single sentence within this window.
// This logic ensures that intra-sentence contributions (N or N-d) dominate.
if (!foundInSameSentenceInWindow)
{
var pair = new WordPairRelatedness(word1, word2, 0);
// Add flat bonus. Using try-add to prevent adding if pair already has value from intra-sentence in a different window where it was same-sentence.
// Or, simply add to the cumulative sum, which is what the dictionary naturally does if the key exists.
if (cumulativeRelatedness.ContainsKey(pair))
{
cumulativeRelatedness[pair] += interSentenceFlatBonus;
}
else
{
cumulativeRelatedness[pair] = interSentenceFlatBonus;
}
}
}
}
processedSentenceCount++;
UpdateProgressBar((int)((double)processedSentenceCount / allSentences.Count * 50) + 50); // Remaining 50%
}
Console.WriteLine("\nWord Pair Relatedness Calculation Complete.");
// --- Stage 4: Generate Reports ---
Console.WriteLine("\nStage 4: Generating Reports...");
UpdateProgressBar(0);
// 4.1 Word Frequency Report
try
{
using (StreamWriter writer = new StreamWriter(freqReportPath))
{
writer.WriteLine("Token,Frequency");
foreach (var entry in wordFrequency.OrderByDescending(kv => kv.Value))
{
writer.WriteLine($"{entry.Key},{entry.Value}");
}
}
Console.WriteLine($"Word Frequency Report generated: {freqReportPath}");
}
catch (Exception ex)
{
Console.WriteLine($"Error writing Word Frequency Report: {ex.Message}");
}
// 4.2 Word Pair Relatedness Report
try
{
using (StreamWriter writer = new StreamWriter(relatednessReportPath))
{
writer.WriteLine("Word1,Word2,RelatednessScore");
foreach (var entry in cumulativeRelatedness.OrderByDescending(kv => kv.Value))
{
writer.WriteLine($"{entry.Key.Word1},{entry.Key.Word2},{entry.Value}");
}
}
Console.WriteLine($"Word Pair Relatedness Report generated: {relatednessReportPath}");
}
catch (Exception ex)
{
Console.WriteLine($"Error writing Word Pair Relatedness Report: {ex.Message}");
}
UpdateProgressBar(100);
Console.WriteLine("\nAnalysis complete. Press any key to exit.");
Console.ReadKey();
}
}
Explanation of Changes and Logic:
Strict Adherence to Rules:
No Tuples, Use Classes:
Sentence,WordStats,WordPairRelatednessclasses are used for all data storage.LINQ: Used extensively for
OrderByDescending,Max,Skip,Where,Any,ToArray,ToList.Preprocessing:
Regex.Replace(fileContent, @"\b([a-zA-Z]{1,3})\.", "$1_", RegexOptions.IgnoreCase): This specifically handlesetc.toetc_. It looks for word boundaries (\b), then 1-3 letters, then a literal dot. It replaces it with the captured letters ($1) followed by an underscore. This is important to distinguish.from sentence terminators.Regex.Replace(fileContent, @"[^A-Z.]+", " ", RegexOptions.IgnoreCase): This is the primary cleaning step. It replaces any character that is not an uppercase letter (A-Z) and not a literal dot (.) with a single space. This handles removing numbers, commas, semicolons, special symbols, etc. (Note:.is kept for sentence splitting).fileContent.ToUpperInvariant(): Converts everything to uppercase.Regex.Replace(fileContent, @"\s+", " ")andTrim(): Cleans up multiple spaces and trims leading/trailing spaces from the entire cleaned text.
Sentence Splitting:
fileContent.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries)correctly splits by.only.Word Tokenization:
rawSentence.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)followed by.Where(t => Regex.IsMatch(t, @"^[A-Z]+$"))ensures only pure alphabetical words remain as tokens.
Relatedness Logic (
WordPairRelatednessClass and Calculation):WordPairRelatednessClass:Stores
Word1,Word2, andRelatednessScore.Canonicalization in Constructor: Crucially, the constructor ensures
Word1is always alphabetically less thanWord2. This meansnew WordPairRelatedness("B", "A", 0)will store internally as ("A", "B"), making it easier to use as a key in a dictionary where("A", "B")and("B", "A")should represent the same pair.EqualsandGetHashCodeOverrides: Essential forWordPairRelatednessobjects to function correctly as keys in aDictionary. They ensure that if twoWordPairRelatednessinstances represent the same pair (e.g.,(A, B)and(B, A)), the dictionary treats them as the same key, allowing scores to be correctly accumulated.
N(Max Sentence Length): Calculated after all sentences are processed, as required by the deeper analysis rule.Two-Pass Relatedness Calculation:
Pass 1 (Intra-Sentence): Iterates through each sentence and calculates relatedness for pairs within that sentence.
Adjacent (distance 1): Adds
N.Distance
d(1 <d<N): AddsN - d. (This interpretation aligns with the60-kconcept, where higher proximity (smallerd) yields higher relatedness).The scores are summed into
cumulativeRelatednessdictionary.
Pass 2 (Inter-Sentence): Iterates through each sentence as the center of a 7-sentence window.
Collects all unique tokens in that 7-sentence window.
Then iterates over all possible pairs of these unique tokens.
Key Logic: It checks
if (!foundInSameSentenceInWindow). This means if a pair (e.g., "APPLE", "TREE") appears within the 7-sentence window, but never in the same single sentence within that specific window, then a flatinterSentenceFlatBonus(e.g.,5.0) is added to their total relatedness. This is the most reasonable interpretation of "if two words are not in same sentence then consider if the neighbouring sentences are there... in this way we can add the relatedness value." It implies a weaker relatedness for cross-sentence co-occurrence compared to same-sentence proximity.
Progress Bar Simulation:
UpdateProgressBarandSetProgressBarMaxmethods are added.Console.Write("\r...")is used to overwrite the current line, simulating a progress bar.Thread.Sleep(10)is added for a visible delay in the console, mimicking UI updates. In a real WinForms app, you'd updateprogressBar.Valueand callprogressBar.Refresh()orInvalidate().
Output Reports:
Generates two CSV files: one for word frequencies and one for word pair relatedness, both sorted in descending order of their respective values.
How to Use and Test:
Create a New C# Console Application Project:
Open Visual Studio.
File > New > Project.
Select "Console App" (.NET Core or .NET 5/6/7/8).
Name it (e.g.,
QhenomenologyAnalyzer).
Replace
Program.csContent: Delete the default code inProgram.csand paste the entire code provided above into it.Prepare an Input Text File: Create a
.txtfile (e.g.,my_article.txt) with some sample text. Include:Words with
.as part of abbreviations (e.g.,etc.,i.e.).Various punctuation (commas, semicolons, question marks).
Numbers (e.g.,
123,2024).Paragraph breaks (just hit Enter multiple times).
Multiple sentences to test relatedness.
Example:
This is a sample text. It has some words. And numbers like 1234. Etc. and some special symbols; also. Another sentence. With some more words, and etc. Another one. The quick brown fox jumps over the lazy dog. Fox and dog.
Run the Application: Press F5 in Visual Studio or use
dotnet runfrom the terminal in your project directory.Provide Input Path: When prompted, enter the full path to your
my_article.txtfile.Check Output: The program will generate
my_article_WordFrequencyReport.csvandmy_article_WordRelatednessReport.csvin the same directory as your input file.
This program now adheres much more strictly to your detailed rules for preprocessing, tokenization, and the custom relatedness calculations, while using your requested class-based and LINQ-driven programming style.
The full C# program above reads a .txt article, normalizes and cleans its contents, and computes:
-
Word Frequency – how many times each word occurs.
-
Word Relatedness – a relative score based on:
-
Same sentence co-occurrence (+30)
-
Adjacency in sentence (+N where N is max sentence length)
-
Distance within sentence (+i×N/k for distance i in sentence of length k)
-
Neighboring sentences (3 before + 3 after, relative scoring)
-
✅ Output:
-
A frequency report:
article_word_frequency_report.txt -
A relatedness report:
article_word_relatedness_report.txt
📌 Next Steps:
-
You can plug this into your form button
button_ANALYZE_ARTICLE_Click. -
Let me know if you want me to modularize into
WordTokenAnalyzer, add aProgressBar, or export in CSV/XLSX.
private void button_FREQUENCY_DISTRIBUTIONS_ON_WORDSNETS_6_COLUMNS_Click(object sender, EventArgs e)
{
System.Windows.Forms.OpenFileDialog ofd = new System.Windows.Forms.OpenFileDialog
{
Title = "Select CSV file",
Filter = "CSV Files (*.csv)|*.csv"
};
if (ofd.ShowDialog() != System.Windows.Forms.DialogResult.OK)
{
return;
}//END OF ELSE OF System.Windows.Forms.OpenFileDialog ofd = new System.Windows.Forms.OpenFileDialog
string inputPath = ofd.FileName;
string baseDir = System.IO.Path.GetDirectoryName(inputPath);
string inputCsvPath = inputPath;// @"wordnet_data.csv"; // Replace with your actual CSV file path
string outputReportPath = inputPath + "_wordnet_token_frequency_report.txt";// = @"wordnet_token_frequency_report.txt";
var tokenFrequency = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var lines = File.ReadAllLines(inputCsvPath);
foreach (var line in lines.Skip(1)) // Skip header
{
// string[] columns = line.Split('\t'); // Assuming TSV (tab-separated); if CSV use ','
string[] columns = line.Split(','); // Assuming TSV (tab-separated); if CSV use ','
if (columns.Length < 6) continue;
for (int i = 0; i < 6; i++)
{
string content = columns[i];
var tokens = content
.Replace("_", " ")
.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var token in tokens)
{
string cleanToken = token.Trim();
if (string.IsNullOrWhiteSpace(cleanToken))
continue;
if (!tokenFrequency.ContainsKey(cleanToken))
tokenFrequency[cleanToken] = 0;
tokenFrequency[cleanToken]++;
}
}
}
var sortedTokens = tokenFrequency.OrderByDescending(kv => kv.Value);
using (var writer = new StreamWriter(outputReportPath))
{
writer.WriteLine("Token\tFrequency");
foreach (var kv in sortedTokens)
{
writer.WriteLine($"{kv.Key}\t{kv.Value}");
}
}
Console.WriteLine("Token frequency report generated: " + outputReportPath);
System.Windows.Forms.MessageBox.Show("Token frequency report generated: " + outputReportPath);
}// private void button_FREQUENCY_DISTRIBUTIONS_ON_WORDSNETS_6_COLUMNS_Click(object sender, EventArgs e)
//(if necessary use classes and use Linq on the class (dont use the tupules)use this style of programming(as below) strictly and first remove all the abbreviations dots example etc. and convert these to etc_ and no dots to remain there for the abbreviations , remove all commans semicolons questions etc remove all the special symbols remove all nonalphabetical things from the text files contents... then split all the sentences with fullstops... convert all the texts to uppercases , remove all the number like things remove all the non alphabetical symbols (other than the full stop symbols , remove all the paragraphs symbols remove the page break symbols also ) and then split the sentences and then find the frequencies of all the words(tokens) and then do some more things there ... we need the relatedness calculations on the texts ... the relatedness is defined with the concept that if two word tokens are in same sentence then the relatedness values are added +30 for that pair ... if these words pairs are placed adjascent then add +60 to relatedness for this pair , if these two pair of words are within k words away then add 60-k as relatedness values for the word pairs (i have assumed that maximum sentences length is 60 words)... for deeper analysis we can find the maximum length sentence(length is measured with the word count if the max length of the all sentences is N words then in second scanning we can assign the relatedness values) and the relatedness values are to calculated as if two words are just adjascent to each other then add relatedness value for this pair as N (that is max length of sentences) and say the current sentence is k words long then the relatedness incrementwe value is k/N if two words are i words away from each other then increase the relatedness value for these pairs of words as +i*N/k which means relatedness increases for smaller sentences and relatedness is proportioned with the max length of the sentences so relatedness is a relative measure with reference to the whole article and the maximum lengthy sentence governs the compactness of relatedness of word pairs and in this way we can get the frequency of words and relatedness values of each pairs of words in the whole article. if two words are not in same sentence then consider if the neighbouring sentences are there then previous 3 sentences and next 3 sentences are considered where total sentence length is considered as total length of 7 sentences (3 previous sentences +3 next sentences and current sentence) in this way we can add the relatedness value for each pair of the words in any given article(in text file) and in this way we can find the pair of words which are related (descending order of relatedness) report along with descending order of word frequency in any article (.txt file) is necessary to get) we need the progressbar tracking also
private void button_FREQUENCY_DISTRIBUTIONS_ON_WORDSNETS_6_COLUMNS_Click(object sender, EventArgs e)
{
System.Windows.Forms.OpenFileDialog ofd = new System.Windows.Forms.OpenFileDialog
{
Title = "Select CSV file",
Filter = "CSV Files (*.csv)|*.csv"
};
if (ofd.ShowDialog() != System.Windows.Forms.DialogResult.OK)
{
return;
}//END OF ELSE OF System.Windows.Forms.OpenFileDialog ofd = new System.Windows.Forms.OpenFileDialog
string inputPath = ofd.FileName;
string baseDir = System.IO.Path.GetDirectoryName(inputPath);
string inputCsvPath = inputPath;// @"wordnet_data.csv"; // Replace with your actual CSV file path
string outputReportPath = inputPath + "_wordnet_token_frequency_report.txt";// = @"wordnet_token_frequency_report.txt";
var tokenFrequency = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var lines = File.ReadAllLines(inputCsvPath);
foreach (var line in lines.Skip(1)) // Skip header
{
// string[] columns = line.Split('\t'); // Assuming TSV (tab-separated); if CSV use ','
string[] columns = line.Split(','); // Assuming TSV (tab-separated); if CSV use ','
if (columns.Length < 6) continue;
for (int i = 0; i < 6; i++)
{
string content = columns[i];
var tokens = content
.Replace("_", " ")
.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var token in tokens)
{
string cleanToken = token.Trim();
if (string.IsNullOrWhiteSpace(cleanToken))
continue;
if (!tokenFrequency.ContainsKey(cleanToken))
tokenFrequency[cleanToken] = 0;
tokenFrequency[cleanToken]++;
}
}
}
var sortedTokens = tokenFrequency.OrderByDescending(kv => kv.Value);
using (var writer = new StreamWriter(outputReportPath))
{
writer.WriteLine("Token\tFrequency");
foreach (var kv in sortedTokens)
{
writer.WriteLine($"{kv.Key}\t{kv.Value}");
}
}
Console.WriteLine("Token frequency report generated: " + outputReportPath);
System.Windows.Forms.MessageBox.Show("Token frequency report generated: " + outputReportPath);
}// private void button_FREQUENCY_DISTRIBUTIONS_ON_WORDSNETS_6_COLUMNS_Click(object sender, EventArgs e) write a c sharp program to find relatedness report in article and the frequency distributions of words in the article
No comments:
Post a Comment